From 7c5c8a2f4ab3fbc72f852816d5c1bb8b58960325 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 24 Aug 2026 23:18:05 +0200 Subject: [PATCH 01/83] chore: add Renovate config (ported from shapepipe) (#308) * chore: add Renovate config (ported from shapepipe) * chore: gate python bumps behind dashboard approval --- renovate.json | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..6a25bd4d --- /dev/null +++ b/renovate.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "description": [ + "THE ONLY Renovate config file — do not add another. A forgotten .github/renovate.json5 (the original Dependabot->Renovate migration config) shadowed this file for weeks: Renovate's config discovery (and Mend's cached config-file name) preferred it, so edits here were silently ignored while its 14-day minimumReleaseAge broke every uv artifact update (renovatebot/renovate#41624). Deleted 2026-07; if Renovate ever again behaves contrary to this file, first check for a second config file (renovate.json5, .github/renovate.json*, .renovaterc*), then the Mend portal's resolved config.", + "Low-noise dependency updates; uv owns version resolution. pyproject keeps floor ranges and rangeStrategy=update-lockfile means `uv lock` picks versions jointly — cross-package conflicts (e.g. numba capping numpy) resolve silently instead of failing the batch, and un-stick themselves when upstream catches up. No minimumReleaseAge anywhere: any age gate is unenforceable with uv resolution (#41624 — uv resolves transitives past the gate and the artifact step fails); CI running the full suite in the container is the automerge gate.", + "Noise: all non-major updates grouped into one weekly PR that automerges when CI is green; majors are gated behind an explicit dashboard-approval tick; lockFileMaintenance refreshes transitive deps weekly. Security fixes (GitHub + OSV alerts) automerge on green CI without waiting for the weekly schedule.", + "python is guarded: Renovate classes base-image bumps like 3.12 -> 3.14 as 'minor', so the major-only approval rule wouldn't catch it — its own packageRule adds dependencyDashboardApproval so a bump sits as an unchecked dashboard item instead of an immortal PR, until someone deliberately ticks it." + ], + "extends": [ + "config:recommended", + "group:allNonMajor", + "schedule:weekly", + "helpers:pinGitHubActionDigests" + ], + "rangeStrategy": "update-lockfile", + "lockFileMaintenance": { + "enabled": true, + "automerge": true + }, + "osvVulnerabilityAlerts": true, + "vulnerabilityAlerts": { + "automerge": true, + "labels": [ + "security" + ] + }, + "packageRules": [ + { + "matchUpdateTypes": [ + "minor", + "patch", + "pin", + "digest" + ], + "automerge": true + }, + { + "matchUpdateTypes": [ + "major" + ], + "dependencyDashboardApproval": true + }, + { + "matchPackageNames": [ + "python" + ], + "groupName": "python", + "automerge": false, + "dependencyDashboardApproval": true + } + ] +} From 409abb77fb23e4be02893e57931d8308128dbae4 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Tue, 25 Aug 2026 00:14:33 +0200 Subject: [PATCH 02/83] =?UTF-8?q?Remove=20vestigial=20FHP/MK=20NOSHEAR?= =?UTF-8?q?=E2=86=901P=20Tpsf=20substitution=20(#257)=20(#267)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metacal estimator and calibrate_comprehensive_cat.py carried a workaround that overwrote the metacal no-shear reconvolution-PSF size with the 1P value — a real fix for an older ShapePipe catalogue whose no-shear PSF was wrong, now a no-op under the current stack: ngmix builds one magnitude-dilated reconv PSF and reuses it across all metacal types, so NOSHEAR and 1P are bit-identical per object. - calibration.py: drop the overwrite; the no-shear branch simply uses the no-shear reconv-PSF column it already reads. - calibrate_comprehensive_cat.py: drop the override and the now-redundant NGMIX_T_PSF_RECONV_NOSHEAR_orig column. - test_calibration.py: update the estimator-stdout note (no hack line now). No output changes under the current stack. test_calibration.py: 8 passed. Claude-Session: https://claude.ai/code/session_01Xthu9uZC17TsaeXh899fhc Co-authored-by: Claude Opus 4.8 --- scripts/calibration/calibrate_comprehensive_cat.py | 10 ---------- src/sp_validation/calibration.py | 8 -------- src/sp_validation/tests/test_calibration.py | 4 ++-- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 4928df9b..2605338f 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -196,16 +196,6 @@ for key in add_cols: add_cols_data[key] = cat.get_col(dat, key, mask_combined._mask, mask_metacal) -# Keep original NOSHEAR column, override with 1P PSF values (FHP/MK hack) -print( - "FHP/MK hack: explicit copying of the metacal no-shear (updated from 1p)" - + " PSF size" -) -add_cols_data["NGMIX_T_PSF_RECONV_NOSHEAR_orig"] = add_cols_data[ - "NGMIX_T_PSF_RECONV_NOSHEAR" -] -add_cols_data["NGMIX_T_PSF_RECONV_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] - # %% # Additional post-processing columns to write to output cat add_cols_post = [ diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index e37fc4d0..e0ab212e 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -795,14 +795,6 @@ def _read_data(self, data, mask): f"Unsupported shape prefix '{self._prefix}'; only 'NGMIX' is supported" ) - print("FHP/MK hack using p1 PSF for ns in cuts") - indices = np.where(mask)[0] - col_1p = f"{self._prefix}_T_PSF_RECONV_1P" - new_psf = data[col_1p][indices] - - # Overwriting incorrect no-shear PSF size to the one from 1p - ns["Tpsf"] = new_psf - self.m1 = m1 self.p1 = p1 self.m2 = m2 diff --git a/src/sp_validation/tests/test_calibration.py b/src/sp_validation/tests/test_calibration.py index 40621c19..0a66aed5 100644 --- a/src/sp_validation/tests/test_calibration.py +++ b/src/sp_validation/tests/test_calibration.py @@ -296,8 +296,8 @@ def test_metacal_R_matrix_recovers_injected_response(): re-runs with slope 5.0 and confirms R11 tracks it (5.0 != 2.0), so a change that decouples R from the input numbers fails. - NOTE: the estimator prints an 'FHP/MK hack' line and an unweighted / - weighted response line; these are expected stdout, not errors. + NOTE: the estimator prints an unweighted / weighted response line; this + is expected stdout, not an error. """ data, n = _build_ngmix_catalog(slope_11=2.0, slope_22=3.0, step=0.01) mask = np.ones(n, dtype=bool) From 2ae0c8b09048b89deb6407e9c310811b63aaae29 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Sun, 30 Aug 2026 01:51:56 +0200 Subject: [PATCH 03/83] Track the Smokescreen fork's main; lock at b457c99 (#311) The provisional SHA pin predated the fork's merged protocol+packaging PRs (and its commit is now orphaned). The library is changing quickly, so pyproject tracks the fork's main and uv.lock carries the exact commit; updates land via uv lock --upgrade-package smokescreen. The DRAW_SCHEME custody assertion fails closed if an install's draw semantics ever drift from the committed blind. Claude-Session: https://claude.ai/code/session_01G9MahwJEQ1t9EuvUXijmy3 Co-authored-by: Claude Fable 5 --- pyproject.toml | 10 +- uv.lock | 1241 ++++++++++++++++++++++++------------------------ 2 files changed, 625 insertions(+), 626 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c6a68dd0..c2a9aa2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,12 +97,10 @@ dependencies = [ # getdist feature-branch below, which is an external fork we pin for repro.) "shear_psf_leakage @ git+https://github.com/CosmoStat/shear_psf_leakage.git@develop", "skyproj", - # UNIONS-WL fork of DESC Smokescreen, pinned by SHA on the fork's - # packaging branch: it declares pyccl and imports its theory backends - # lazily, so the install closure is CCL-only. Provisional pin — swapped to the - # fork's release tag once the fork packaging PRs merge. Git pin only; - # nothing is published to PyPI. - "smokescreen @ git+https://github.com/UNIONS-WL/Smokescreen@588a6b9b26560bd5ba3dd5ba342f3c40152644f9", + # UNIONS-WL fork of DESC Smokescreen. Tracks the fork's main; uv.lock + # freezes the exact commit, so updates land via + # `uv lock --upgrade-package smokescreen`. Git only; not on PyPI. + "smokescreen @ git+https://github.com/UNIONS-WL/Smokescreen@main", "statsmodels", "treecorr>=5.0", "tqdm", diff --git a/uv.lock b/uv.lock index bef8072e..deedc1cf 100644 --- a/uv.lock +++ b/uv.lock @@ -14,9 +14,9 @@ name = "adjusttext" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/5c/496e506ad3313664df24a79f801719cadcd62af5999fcb299c9b08ff0d4b/adjusttext-1.4.0.tar.gz", hash = "sha256:1f73860ced8cccce3f85ee6989ca133c2579b67a7453f63dbeb38f39bf123154", size = 15852, upload-time = "2026-06-08T16:48:32.726Z" } wheels = [ @@ -55,8 +55,8 @@ name = "anyio" version = "4.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } wheels = [ @@ -68,7 +68,7 @@ name = "argon2-cffi" version = "25.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argon2-cffi-bindings", marker = "sys_platform == 'linux'" }, + { name = "argon2-cffi-bindings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } wheels = [ @@ -80,7 +80,7 @@ name = "argon2-cffi-bindings" version = "25.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "sys_platform == 'linux'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } wheels = [ @@ -117,8 +117,8 @@ name = "arrow" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "tzdata", marker = "sys_platform == 'linux'" }, + { name = "python-dateutil" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } wheels = [ @@ -139,11 +139,11 @@ name = "astropy" version = "8.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy-iers-data", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyerfa", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "astropy-iers-data" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyerfa" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/c4/21be4313ddfde5f60e0607fd307f367b9e0f0bf153a89b10cbd036dd8cfd/astropy-8.0.1.tar.gz", hash = "sha256:45ca31d5b91fa294cd590a4791a32db94de7f9c8a343155f4d5877baa82351da", size = 7152500, upload-time = "2026-07-05T07:24:48.482Z" } wheels = [ @@ -157,8 +157,8 @@ name = "astropy-healpix" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/bb/fb3dcca212b1d57ed26b33eabb9e1cc18fcadbfc2f2976b20530b7d80b90/astropy_healpix-2.0.0.tar.gz", hash = "sha256:6ce7646dc5b1c09a7ed24b1f23d3e2fce8e3570387220d26a24487ab001b3b0b", size = 112064, upload-time = "2026-06-29T20:02:46.407Z" } wheels = [ @@ -220,8 +220,8 @@ name = "beautifulsoup4" version = "4.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "soupsieve", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "soupsieve" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } wheels = [ @@ -233,7 +233,7 @@ name = "bleach" version = "6.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "webencodings", marker = "sys_platform == 'linux'" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/3c/e12ac860709702bd5ebeb9b56a4fe334f1001246ee1b8f2b7ee28912df7d/bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452", size = 204857, upload-time = "2026-06-05T13:01:13.734Z" } wheels = [ @@ -242,7 +242,7 @@ wheels = [ [package.optional-dependencies] css = [ - { name = "tinycss2", marker = "sys_platform == 'linux'" }, + { name = "tinycss2" }, ] [[package]] @@ -250,14 +250,14 @@ name = "blosc2" version = "4.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx", extra = ["http2"], marker = "sys_platform == 'linux'" }, - { name = "msgpack", marker = "sys_platform == 'linux'" }, - { name = "ndindex", marker = "sys_platform == 'linux'" }, - { name = "numexpr", marker = "platform_machine != 'wasm32' and sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "rich", marker = "sys_platform == 'linux'" }, - { name = "threadpoolctl", marker = "platform_machine != 'wasm32' and sys_platform == 'linux'" }, + { name = "httpx", extra = ["http2"] }, + { name = "msgpack" }, + { name = "ndindex" }, + { name = "numexpr", marker = "platform_machine != 'wasm32'" }, + { name = "numpy" }, + { name = "pydantic" }, + { name = "rich" }, + { name = "threadpoolctl", marker = "platform_machine != 'wasm32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c1/26/c5b32a553f002e91886213ec91ff56a2dd8555e5e78eb4c289addb208642/blosc2-4.8.0.tar.gz", hash = "sha256:9789c3248052be59e63d6debafec4cd59d0eb4a6fd1cbaced36865d5d40ffa63", size = 5681856, upload-time = "2026-07-10T09:38:33.592Z" } wheels = [ @@ -278,12 +278,12 @@ name = "cadcutils" version = "1.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "distro", marker = "sys_platform == 'linux'" }, - { name = "lxml", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyopenssl", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "distro" }, + { name = "lxml" }, + { name = "packaging" }, + { name = "pyopenssl" }, + { name = "requests" }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fe/44/4964823c72c9e6a93e26de0118cead37fa3c3600b4677f0c06788c903c80/cadcutils-1.6.2.tar.gz", hash = "sha256:6e2d7822756d48a363bc5f857cd717f80aab760554df86518b957f8826a7a906", size = 94673, upload-time = "2026-06-22T17:35:47.303Z" } wheels = [ @@ -295,10 +295,10 @@ name = "camb" version = "1.6.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "sympy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "scipy" }, + { name = "sympy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/ad/f2e6446fbd94f7adc1474e1f4e40d2e7fa7e5e7143640cb8557d492f0a73/camb-1.6.6.tar.gz", hash = "sha256:9856202a5c05570256e52377b20431891c7b08b2e9c334e141fd08d2a085516f", size = 799344, upload-time = "2026-03-11T16:46:55.819Z" } wheels = [ @@ -320,7 +320,7 @@ name = "cffi" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy' and sys_platform == 'linux'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } wheels = [ @@ -429,12 +429,12 @@ name = "clmm" version = "1.16.10" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "qp-prob", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "healpy" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "qp-prob" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/73/7ff76d487cdbcddb9c80ec03313db1b11f48fe4fcb5e1609b6eca5a73159/clmm-1.16.10.tar.gz", hash = "sha256:48273784419c038a89619b798a140153ad1e451390ff56074f3b6825c91ebe32", size = 72475, upload-time = "2025-12-04T17:30:11.47Z" } wheels = [ @@ -473,7 +473,7 @@ name = "conda-inject" version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/a8/8dc86113c65c949cc72d651461d6e4c544b3302a85ed14a5298829e6a419/conda_inject-1.3.2.tar.gz", hash = "sha256:0b8cde8c47998c118d8ff285a04977a3abcf734caf579c520fca469df1cd0aac", size = 3635, upload-time = "2024-05-27T12:20:58.873Z" } wheels = [ @@ -500,7 +500,7 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -541,12 +541,12 @@ name = "cosmo-numba" version = "1.0.1.dev6+g188d272c6" source = { git = "https://github.com/aguinot/cosmo-numba.git?rev=main#188d272c67d6d699d7cff7a7a53ded8dec759bb0" } dependencies = [ - { name = "mpmath", marker = "sys_platform == 'linux'" }, - { name = "numba", marker = "sys_platform == 'linux'" }, - { name = "numbaquadpack", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "rocket-fft", marker = "sys_platform == 'linux'" }, - { name = "sympy", marker = "sys_platform == 'linux'" }, + { name = "mpmath" }, + { name = "numba" }, + { name = "numbaquadpack" }, + { name = "numpy" }, + { name = "rocket-fft" }, + { name = "sympy" }, ] [[package]] @@ -554,7 +554,7 @@ name = "cosmology" version = "2022.10.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/38/04099bb2b6a626bd69a3f117afc60f2c47c08899e0730e0e48c898bf4745/cosmology-2022.10.9.tar.gz", hash = "sha256:0c2857c9bf1fdd09f1f11ab5765df0389a4101f3a900fa11251c3f37696f02d4", size = 8488, upload-time = "2022-10-10T10:18:16.229Z" } wheels = [ @@ -615,7 +615,7 @@ name = "cryptography" version = "49.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'linux'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } wheels = [ @@ -659,18 +659,18 @@ name = "cs-util" version = "0.2.2" source = { git = "https://github.com/CosmoStat/cs_util.git?rev=develop#bedcdbf50ac57fc4d0b72a62b251373cd1807824" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "datetime", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "keyring", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "swig", marker = "sys_platform == 'linux'" }, - { name = "vos", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "camb" }, + { name = "datetime" }, + { name = "healpy" }, + { name = "healsparse" }, + { name = "keyring" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pyccl" }, + { name = "scipy" }, + { name = "swig" }, + { name = "vos" }, ] [[package]] @@ -687,13 +687,13 @@ name = "dask" version = "2026.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "sys_platform == 'linux'" }, - { name = "cloudpickle", marker = "sys_platform == 'linux'" }, - { name = "fsspec", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "partd", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "toolz", marker = "sys_platform == 'linux'" }, + { name = "click" }, + { name = "cloudpickle" }, + { name = "fsspec" }, + { name = "packaging" }, + { name = "partd" }, + { name = "pyyaml" }, + { name = "toolz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/b5/aa50877159f1efd65a744107e0662a15d2b80fa0c427d980313367f49493/dask-2026.7.0.tar.gz", hash = "sha256:b039970642ce97a063070bae2763dada8aae27e3cbd8ff6d894072f920a1e252", size = 11548914, upload-time = "2026-07-06T16:58:36.496Z" } wheels = [ @@ -702,7 +702,7 @@ wheels = [ [package.optional-dependencies] array = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] [[package]] @@ -710,11 +710,11 @@ name = "dask-image" version = "2026.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pims", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, + { name = "dask", extra = ["array"] }, + { name = "numpy" }, + { name = "pims" }, + { name = "scipy" }, + { name = "tifffile" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/49/e592a13a5e1efdcdb8f1faab7c4e309c61648792e276f9ced5fb79381b33/dask_image-2026.5.0.tar.gz", hash = "sha256:ed6b462277e691b2c12b0890ba801a0f9a00cc1894b0aa71b195a7a1419b2b00", size = 80457, upload-time = "2026-05-27T14:05:57.383Z" } wheels = [ @@ -726,8 +726,8 @@ name = "datetime" version = "6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pytz", marker = "sys_platform == 'linux'" }, - { name = "zope-interface", marker = "sys_platform == 'linux'" }, + { name = "pytz" }, + { name = "zope-interface" }, ] sdist = { url = "https://files.pythonhosted.org/packages/77/32/decbfd165e9985ba9d8c2d34a39afe5aeba2fc3fe390eb6e9ef1aab98fa8/datetime-6.0.tar.gz", hash = "sha256:c1514936d2f901e10c8e08d83bf04e6c9dbd7ca4f244da94fec980980a3bc4d5", size = 64167, upload-time = "2025-11-25T08:00:34.586Z" } wheels = [ @@ -769,7 +769,7 @@ name = "deprecated" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt", marker = "sys_platform == 'linux'" }, + { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ @@ -817,7 +817,7 @@ name = "donfig" version = "0.8.1.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } wheels = [ @@ -838,7 +838,7 @@ name = "ducc0" version = "0.41.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cf/e7/849dd1a079b994184fa53905d47278f86d0dd3dcfb6ff2b0e3eb7d6a1c1b/ducc0-0.41.0.tar.gz", hash = "sha256:bac084745bbdb243482a4aec3ecc857bdd46faec298c67e48bc0bb90350dbabd", size = 353129, upload-time = "2026-03-26T18:56:04.669Z" } wheels = [ @@ -857,7 +857,7 @@ name = "emcee" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/53/1045ee878cb24281387079f8ee4f0ade1622c6aae1ed1fd91a53e4fa5b19/emcee-3.1.6.tar.gz", hash = "sha256:11af4daf6ab8f9ca69681e3c29054665db7bbd87fd4eb8e437d2c3a1248c637d", size = 2871117, upload-time = "2024-04-19T10:03:19.555Z" } wheels = [ @@ -887,7 +887,7 @@ name = "fitsio" version = "1.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/34/6b/5658778e070faaa194f1a1133527cedfa948d173daed29d61f25c5487ab6/fitsio-1.4.1.tar.gz", hash = "sha256:2b2bf0bfd37790114b44283931c59a14b1fec25a653bf65c4fc555b8d3b24de9", size = 4148217, upload-time = "2026-07-10T21:21:31.585Z" } wheels = [ @@ -906,7 +906,7 @@ name = "flt" version = "2026.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/80/66/538256582e373d81a944c0ee3a12fd0e4dd0f39d10136353c4ce914d7061/flt-2026.1.tar.gz", hash = "sha256:4bf2818d4a9137f5f715fd2ff13acfbf0175639373155cd89ea2b788edf00093", size = 12817, upload-time = "2026-03-20T22:47:29.754Z" } wheels = [ @@ -977,11 +977,11 @@ name = "getdist" version = "1.6.1" source = { git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5#113cd22a9a0d013b6f72fe734be81f260f3d3be5" } dependencies = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "scipy" }, ] [[package]] @@ -989,7 +989,7 @@ name = "gitdb" version = "4.0.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "smmap", marker = "sys_platform == 'linux'" }, + { name = "smmap" }, ] sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } wheels = [ @@ -1001,7 +1001,7 @@ name = "gitpython" version = "3.1.50" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "gitdb", marker = "sys_platform == 'linux'" }, + { name = "gitdb" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } wheels = [ @@ -1013,11 +1013,11 @@ name = "glass" version = "2025.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cosmology", marker = "sys_platform == 'linux'" }, - { name = "healpix", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "transformcl", marker = "sys_platform == 'linux'" }, + { name = "cosmology" }, + { name = "healpix" }, + { name = "healpy" }, + { name = "numpy" }, + { name = "transformcl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/d6/26cd58e75834671259f26b9287a584d726e6023da35ddf7ca3b7f2c393fb/glass-2025.1.tar.gz", hash = "sha256:7b1aa2394e16010f7f1b4243f49e7e12d7a4dd28fcbf3e3f7cf25ce4905a8615", size = 48533, upload-time = "2025-02-21T18:43:48.046Z" } wheels = [ @@ -1029,9 +1029,9 @@ name = "glass-ext-camb" version = "2023.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "glass", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "camb" }, + { name = "glass" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/1a/18bb47c0f7cc267c2f8c95273055b82b1975e4dccbfea6051e82a4d1f513/glass.ext.camb-2023.6.tar.gz", hash = "sha256:f7b6e521b2d45d3b68400353ce6ab1747c772631832a257567de028aa2060509", size = 2989, upload-time = "2023-06-30T22:27:31.511Z" } wheels = [ @@ -1113,8 +1113,8 @@ name = "h2" version = "4.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "hpack", marker = "sys_platform == 'linux'" }, - { name = "hyperframe", marker = "sys_platform == 'linux'" }, + { name = "hpack" }, + { name = "hyperframe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } wheels = [ @@ -1126,7 +1126,7 @@ name = "h5py" version = "3.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } wheels = [ @@ -1153,7 +1153,7 @@ name = "healpix" version = "2025.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/1a/c1af658a2d06cdd2734b085b0eee34d51646f091ed13b1669d3a596de57c/healpix-2025.1.tar.gz", hash = "sha256:733f16f30c15d911c8209f4045f0b0934e5fcafa63b8b39d62e425a036adccea", size = 22744, upload-time = "2025-07-07T16:38:42.402Z" } wheels = [ @@ -1168,8 +1168,8 @@ name = "healpy" version = "1.19.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e3/4e/6f5328f375f88c8c38314f24f82b7e1486ec0358c557779500382db1503c/healpy-1.19.0.tar.gz", hash = "sha256:28e839cb885a23d36c77fc3423a3cb9271a07fda94085bd12fc329f941130ec5", size = 4075006, upload-time = "2025-12-02T08:27:19.766Z" } wheels = [ @@ -1192,8 +1192,8 @@ name = "healsparse" version = "1.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "hpgeom" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e3/33/36520faabae128fe9a4a5cbeb0916be864d2eb34cf7410d52dc532af044e/healsparse-1.13.0.tar.gz", hash = "sha256:c21e4eb075c686bd306bdc1b40e039134c4ca60ccfa4e7e73845c107a5b16544", size = 133589, upload-time = "2026-07-07T18:33:05.481Z" } wheels = [ @@ -1214,7 +1214,7 @@ name = "hpgeom" version = "1.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/72/27640de7d85a566b8ecd27e7894f371a19669c56c268756e18111a137df8/hpgeom-1.5.4.tar.gz", hash = "sha256:85ac73e267c11f3f248920d3541a0e6ffab821cae54f0d44414d1939dfb91200", size = 153906, upload-time = "2026-02-24T19:03:01.954Z" } wheels = [ @@ -1240,8 +1240,8 @@ name = "httpcore" version = "1.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "h11", marker = "sys_platform == 'linux'" }, + { name = "certifi" }, + { name = "h11" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ @@ -1253,10 +1253,10 @@ name = "httpx" version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform == 'linux'" }, - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "httpcore", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ @@ -1265,7 +1265,7 @@ wheels = [ [package.optional-dependencies] http2 = [ - { name = "h2", marker = "sys_platform == 'linux'" }, + { name = "h2" }, ] [[package]] @@ -1300,8 +1300,8 @@ name = "imageio" version = "2.37.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "pillow" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/84/93bcd1300216ea50811cee96873b84a1bebf8d0489ffaf7f2a3756bab866/imageio-2.37.3.tar.gz", hash = "sha256:bbb37efbfc4c400fcd534b367b91fcd66d5da639aaa138034431a1c5e0a41451", size = 389673, upload-time = "2026-03-09T11:31:12.573Z" } wheels = [ @@ -1338,7 +1338,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "sys_platform == 'linux'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -1368,18 +1368,18 @@ name = "ipykernel" version = "7.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "comm", marker = "sys_platform == 'linux'" }, - { name = "debugpy", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, - { name = "nest-asyncio2", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio2" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/c4/e4a38f579de4225a561305666f7541cdabb30075def2aa1ac17bd73c1fb5/ipykernel-7.3.0.tar.gz", hash = "sha256:9acaaaf97d16355166e4085afe9d225bfbdf2b7ef520f9df3be8f2b248275e09", size = 184899, upload-time = "2026-06-10T08:41:25.481Z" } wheels = [ @@ -1391,16 +1391,16 @@ name = "ipython" version = "9.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "decorator", marker = "sys_platform == 'linux'" }, - { name = "ipython-pygments-lexers", marker = "sys_platform == 'linux'" }, - { name = "jedi", marker = "sys_platform == 'linux'" }, - { name = "matplotlib-inline", marker = "sys_platform == 'linux'" }, - { name = "pexpect", marker = "sys_platform == 'linux'" }, - { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "stack-data", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect" }, + { name = "prompt-toolkit" }, + { name = "psutil" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -1421,7 +1421,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1433,11 +1433,11 @@ name = "ipywidgets" version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "comm", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-widgets", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "widgetsnbextension", marker = "sys_platform == 'linux'" }, + { name = "comm" }, + { name = "ipython" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4c/ae/c5ce1edc1afe042eadb445e95b0671b03cee61895264357956e61c0d2ac0/ipywidgets-8.1.8.tar.gz", hash = "sha256:61f969306b95f85fba6b6986b7fe45d73124d1d9e3023a8068710d47a22ea668", size = 116739, upload-time = "2025-11-01T21:18:12.393Z" } wheels = [ @@ -1449,7 +1449,7 @@ name = "isoduration" version = "20.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "arrow", marker = "sys_platform == 'linux'" }, + { name = "arrow" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" } wheels = [ @@ -1461,7 +1461,7 @@ name = "jaraco-classes" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools", marker = "sys_platform == 'linux'" }, + { name = "more-itertools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } wheels = [ @@ -1482,7 +1482,7 @@ name = "jaraco-functools" version = "4.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools", marker = "sys_platform == 'linux'" }, + { name = "more-itertools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/36/cf/ea4ef2920830dea3f5ab2ea4da6fb67724e6dca80ee2553788c3607243d0/jaraco_functools-4.5.0.tar.gz", hash = "sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03", size = 20272, upload-time = "2026-05-15T21:34:10.025Z" } wheels = [ @@ -1494,7 +1494,7 @@ name = "jedi" version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "parso", marker = "sys_platform == 'linux'" }, + { name = "parso" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } wheels = [ @@ -1515,7 +1515,7 @@ name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe", marker = "sys_platform == 'linux'" }, + { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ @@ -1545,7 +1545,7 @@ name = "jsonargparse" version = "4.49.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/87/0b/f012466077d803b546955bf827ab30a320ad1550e3167195432cc2d6bfad/jsonargparse-4.49.0.tar.gz", hash = "sha256:9e691a09d937fb4c6bc1f6d3bf6c3546efa03381ea1e64ee10759095f1b14da7", size = 125237, upload-time = "2026-05-15T06:48:44.807Z" } wheels = [ @@ -1554,8 +1554,8 @@ wheels = [ [package.optional-dependencies] signatures = [ - { name = "docstring-parser", marker = "sys_platform == 'linux'" }, - { name = "typeshed-client", marker = "sys_platform == 'linux'" }, + { name = "docstring-parser" }, + { name = "typeshed-client" }, ] [[package]] @@ -1572,10 +1572,10 @@ name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "sys_platform == 'linux'" }, - { name = "jsonschema-specifications", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "rpds-py", marker = "sys_platform == 'linux'" }, + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ @@ -1584,15 +1584,15 @@ wheels = [ [package.optional-dependencies] format-nongpl = [ - { name = "fqdn", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "isoduration", marker = "sys_platform == 'linux'" }, - { name = "jsonpointer", marker = "sys_platform == 'linux'" }, - { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3987-syntax", marker = "sys_platform == 'linux'" }, - { name = "uri-template", marker = "sys_platform == 'linux'" }, - { name = "webcolors", marker = "sys_platform == 'linux'" }, + { name = "fqdn" }, + { name = "idna" }, + { name = "isoduration" }, + { name = "jsonpointer" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator" }, + { name = "rfc3987-syntax" }, + { name = "uri-template" }, + { name = "webcolors" }, ] [[package]] @@ -1600,7 +1600,7 @@ name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "referencing", marker = "sys_platform == 'linux'" }, + { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ @@ -1612,12 +1612,12 @@ name = "jupyter" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "ipywidgets", marker = "sys_platform == 'linux'" }, - { name = "jupyter-console", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "notebook", marker = "sys_platform == 'linux'" }, + { name = "ipykernel" }, + { name = "ipywidgets" }, + { name = "jupyter-console" }, + { name = "jupyterlab" }, + { name = "nbconvert" }, + { name = "notebook" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/f3/af28ea964ab8bc1e472dba2e82627d36d470c51f5cd38c37502eeffaa25e/jupyter-1.1.1.tar.gz", hash = "sha256:d55467bceabdea49d7e3624af7e33d59c37fff53ed3a350e1ac957bed731de7a", size = 5714959, upload-time = "2024-08-30T07:15:48.299Z" } wheels = [ @@ -1629,8 +1629,8 @@ name = "jupyter-builder" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/df/db5efc4e28803a1421350445e53d85ec571a4e6928e3524ccc39f4e16584/jupyter_builder-1.1.0.tar.gz", hash = "sha256:b996e8af616900f18724fa34883169d869be2205497940bec78a3a1031eb897d", size = 971142, upload-time = "2026-07-11T07:24:02.4Z" } wheels = [ @@ -1642,12 +1642,12 @@ name = "jupyter-client" version = "8.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/dc/5512503b088997c2250b8bf18258fba9d9ce5ead641183700960d3c9d342/jupyter_client-8.9.1.tar.gz", hash = "sha256:a58f730dd9e728ba16ba1d62ebccf7ffe1ebbdbce4e95cfae941b7321ae1f4fa", size = 359256, upload-time = "2026-06-09T13:15:01.033Z" } wheels = [ @@ -1659,14 +1659,14 @@ name = "jupyter-console" version = "6.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "ipython", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "prompt-toolkit", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "ipykernel" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "pyzmq" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/2d/e2fd31e2fc41c14e2bcb6c976ab732597e907523f6b2420305f9fc7fdbdb/jupyter_console-6.6.3.tar.gz", hash = "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539", size = 34363, upload-time = "2023-03-06T14:13:31.02Z" } wheels = [ @@ -1678,8 +1678,8 @@ name = "jupyter-core" version = "5.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "platformdirs" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } wheels = [ @@ -1691,14 +1691,14 @@ name = "jupyter-events" version = "0.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonschema", extra = ["format-nongpl"], marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "python-json-logger", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "rfc3339-validator", marker = "sys_platform == 'linux'" }, - { name = "rfc3986-validator", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "jsonschema", extra = ["format-nongpl"] }, + { name = "packaging" }, + { name = "python-json-logger" }, + { name = "pyyaml" }, + { name = "referencing" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/f8/475c4241b2b75af0deaae453ed003c6c851766dbc44d332d8baf245dc931/jupyter_events-0.12.1.tar.gz", hash = "sha256:faff25f77218335752f35f23c5fe6e4a392a7bd99a5939ccb9b8fbf594636cf3", size = 62854, upload-time = "2026-04-20T23:17:50.66Z" } wheels = [ @@ -1710,7 +1710,7 @@ name = "jupyter-lsp" version = "2.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server" }, ] sdist = { url = "https://files.pythonhosted.org/packages/36/ff/1e4a61f5170a9a1d978f3ac3872449de6c01fc71eaf89657824c878b1549/jupyter_lsp-2.3.1.tar.gz", hash = "sha256:fdf8a4aa7d85813976d6e29e95e6a2c8f752701f926f2715305249a3829805a6", size = 55677, upload-time = "2026-04-02T08:10:06.749Z" } wheels = [ @@ -1722,23 +1722,23 @@ name = "jupyter-server" version = "2.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform == 'linux'" }, - { name = "argon2-cffi", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyter-events", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server-terminals", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "prometheus-client", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "send2trash", marker = "sys_platform == 'linux'" }, - { name = "terminado", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, - { name = "websocket-client", marker = "sys_platform == 'linux'" }, + { name = "anyio" }, + { name = "argon2-cffi" }, + { name = "jinja2" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "jupyter-events" }, + { name = "jupyter-server-terminals" }, + { name = "nbconvert" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "prometheus-client" }, + { name = "pyzmq" }, + { name = "send2trash" }, + { name = "terminado" }, + { name = "tornado" }, + { name = "traitlets" }, + { name = "websocket-client" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6b/dc/db3a582633170186f8c8b31298d7eb26ad0eb031a1f53476c258b64eed05/jupyter_server-2.20.0.tar.gz", hash = "sha256:b5778ba337d8015a3dc2b80803ecdd5ac18d3797fddf61a50ea5fb472b4ebe14", size = 756523, upload-time = "2026-06-17T12:09:09.435Z" } wheels = [ @@ -1750,7 +1750,7 @@ name = "jupyter-server-terminals" version = "0.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "terminado", marker = "sys_platform == 'linux'" }, + { name = "terminado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } wheels = [ @@ -1762,19 +1762,19 @@ name = "jupyterlab" version = "4.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "async-lru", marker = "sys_platform == 'linux'" }, - { name = "httpx", marker = "sys_platform == 'linux'" }, - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-builder", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyter-lsp", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-server", marker = "sys_platform == 'linux'" }, - { name = "notebook-shim", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "async-lru" }, + { name = "httpx" }, + { name = "ipykernel" }, + { name = "jinja2" }, + { name = "jupyter-builder" }, + { name = "jupyter-core" }, + { name = "jupyter-lsp" }, + { name = "jupyter-server" }, + { name = "jupyterlab-server" }, + { name = "notebook-shim" }, + { name = "packaging" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/2a/d6af53bfd45a43a5bfe7e40ba47ee7a8921a807daf4bb708e3a295bbb54d/jupyterlab-4.6.1.tar.gz", hash = "sha256:75315982ed28427edaa62bb85eadb5105e4043a757643c910efd787fe6ed0837", size = 28179125, upload-time = "2026-06-29T12:48:45.402Z" } wheels = [ @@ -1795,13 +1795,13 @@ name = "jupyterlab-server" version = "2.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "json5", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "babel" }, + { name = "jinja2" }, + { name = "json5" }, + { name = "jsonschema" }, + { name = "jupyter-server" }, + { name = "packaging" }, + { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d6/2c/90153f189e421e93c4bb4f9e3f59802a1f01abd2ac5cf40b152d7f735232/jupyterlab_server-2.28.0.tar.gz", hash = "sha256:35baa81898b15f93573e2deca50d11ac0ae407ebb688299d3a5213265033712c", size = 76996, upload-time = "2025-10-22T13:59:18.37Z" } wheels = [ @@ -1822,11 +1822,11 @@ name = "jupytext" version = "1.19.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/52/e014296ac8f40ca783aeb73dae52e65edbb0eaae0dcdc1ea41bfaa8aebf7/jupytext-1.19.4.tar.gz", hash = "sha256:739bcd4bc12aa4fe298a38017cdb5ae27b08a6ba3a5470728d2fe9e04b155db1", size = 4581977, upload-time = "2026-06-21T21:48:58.32Z" } wheels = [ @@ -1838,11 +1838,11 @@ name = "keyring" version = "25.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jaraco-classes", marker = "sys_platform == 'linux'" }, - { name = "jaraco-context", marker = "sys_platform == 'linux'" }, - { name = "jaraco-functools", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney" }, + { name = "secretstorage" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } wheels = [ @@ -1931,9 +1931,9 @@ name = "lenspack" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/73/de/18cfc64028334a1907d44ba2b40e233a2c7a06d7c1e827acfd1785c0b293/lenspack-1.0.0.tar.gz", hash = "sha256:e528a5409473eb996b39695cccb744db067341eb5fac6f240cd8d0aa669c85ec", size = 32632, upload-time = "2020-09-04T08:33:53.267Z" } wheels = [ @@ -1961,11 +1961,11 @@ name = "lmfit" version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "asteval", marker = "sys_platform == 'linux'" }, - { name = "dill", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "uncertainties", marker = "sys_platform == 'linux'" }, + { name = "asteval" }, + { name = "dill" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "uncertainties" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/e5/a35942aed2de95e228728c34609b51fe3ec9182398eac50d288eef313aa2/lmfit-1.3.4.tar.gz", hash = "sha256:3c22c28c43f717f6c5b4a3bd81e893a2149739c26a592c046f2e33c23cfbe497", size = 630720, upload-time = "2025-07-19T20:09:01.876Z" } wheels = [ @@ -1986,8 +1986,8 @@ name = "lsstdesc-coord" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "future", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "future" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/e0/6806df4cfa54927a2d8e425407611b4fc031dd125bc952b4a31deb743305/lsstdesc_coord-1.3.1.tar.gz", hash = "sha256:60f878c29e1f30a9b50bf60dca3c466dc9cfb2cbc71f0a27c575ced969de57ab", size = 41647, upload-time = "2026-02-13T18:37:32.611Z" } wheels = [ @@ -2059,7 +2059,7 @@ name = "markdown-it-py" version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl", marker = "sys_platform == 'linux'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ @@ -2109,15 +2109,15 @@ name = "matplotlib" version = "3.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy", marker = "sys_platform == 'linux'" }, - { name = "cycler", marker = "sys_platform == 'linux'" }, - { name = "fonttools", marker = "sys_platform == 'linux'" }, - { name = "kiwisolver", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "pyparsing", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/24/080c99d223d158d3a8902769269ab6da5b50f7a0e6e072513907e02b7a6c/matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57", size = 33251176, upload-time = "2026-06-12T02:29:15.508Z" } wheels = [ @@ -2143,7 +2143,7 @@ name = "matplotlib-inline" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } wheels = [ @@ -2155,7 +2155,7 @@ name = "mdit-py-plugins" version = "0.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, + { name = "markdown-it-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } wheels = [ @@ -2255,12 +2255,12 @@ name = "myst-parser" version = "5.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "mdit-py-plugins", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/dc/603751677fff302f34396e206b610f556a59d7fe58b9a2145f54e96b48e8/myst_parser-5.1.0.tar.gz", hash = "sha256:ab69322dc6719dcc7f296479dbb70181b66df6ed315064f92dbc85c0e1bf2f02", size = 101182, upload-time = "2026-05-13T09:38:19.361Z" } wheels = [ @@ -2272,10 +2272,10 @@ name = "nbclient" version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbformat" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } wheels = [ @@ -2287,20 +2287,20 @@ name = "nbconvert" version = "7.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "bleach", extra = ["css"], marker = "sys_platform == 'linux'" }, - { name = "defusedxml", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab-pygments", marker = "sys_platform == 'linux'" }, - { name = "markupsafe", marker = "sys_platform == 'linux'" }, - { name = "mistune", marker = "sys_platform == 'linux'" }, - { name = "nbclient", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pandocfilters", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "beautifulsoup4" }, + { name = "bleach", extra = ["css"] }, + { name = "defusedxml" }, + { name = "jinja2" }, + { name = "jupyter-core" }, + { name = "jupyterlab-pygments" }, + { name = "markupsafe" }, + { name = "mistune" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "pandocfilters" }, + { name = "pygments" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } wheels = [ @@ -2312,10 +2312,10 @@ name = "nbformat" version = "5.10.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fastjsonschema", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } wheels = [ @@ -2364,19 +2364,19 @@ name = "notebook" version = "5.7.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipykernel", marker = "sys_platform == 'linux'" }, - { name = "ipython-genutils", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jupyter-client", marker = "sys_platform == 'linux'" }, - { name = "jupyter-core", marker = "sys_platform == 'linux'" }, - { name = "nbconvert", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "prometheus-client", marker = "sys_platform == 'linux'" }, - { name = "pyzmq", marker = "sys_platform == 'linux'" }, - { name = "send2trash", marker = "sys_platform == 'linux'" }, - { name = "terminado", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "traitlets", marker = "sys_platform == 'linux'" }, + { name = "ipykernel" }, + { name = "ipython-genutils" }, + { name = "jinja2" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbconvert" }, + { name = "nbformat" }, + { name = "prometheus-client" }, + { name = "pyzmq" }, + { name = "send2trash" }, + { name = "terminado" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6e/22/b5dcce67559d63d0f22e46d806305710808c698a1b91c07eb09e389785e0/notebook-5.7.4.tar.gz", hash = "sha256:d908673a4010787625c8952e91a22adf737db031f2aa0793ad92f6558918a74a", size = 13389469, upload-time = "2018-12-17T10:06:00.805Z" } wheels = [ @@ -2388,7 +2388,7 @@ name = "notebook-shim" version = "0.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, + { name = "jupyter-server" }, ] sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } wheels = [ @@ -2400,8 +2400,8 @@ name = "numba" version = "0.66.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "llvmlite", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "llvmlite" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/a0/570e3dc53e5602b49108f62a13e529f1eec8bfc7ef37d49c825924dcf546/numba-0.66.0.tar.gz", hash = "sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363", size = 2806181, upload-time = "2026-07-01T23:12:46.36Z" } wheels = [ @@ -2425,8 +2425,8 @@ name = "numcodecs" version = "0.16.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } wheels = [ @@ -2443,7 +2443,7 @@ name = "numexpr" version = "2.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/2f/fdba158c9dbe5caca9c3eca3eaffffb251f2fb8674bf8e2d0aed5f38d319/numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b", size = 119400, upload-time = "2025-10-13T16:17:27.351Z" } wheels = [ @@ -2502,7 +2502,7 @@ name = "numpydoc" version = "1.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } wheels = [ @@ -2514,7 +2514,7 @@ name = "opencv-python-headless" version = "5.0.0.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } wheels = [ @@ -2538,10 +2538,10 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "python-dateutil", marker = "sys_platform == 'linux'" }, - { name = "pytz", marker = "sys_platform == 'linux'" }, - { name = "tzdata", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2590,8 +2590,8 @@ name = "partd" version = "1.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "locket", marker = "sys_platform == 'linux'" }, - { name = "toolz", marker = "sys_platform == 'linux'" }, + { name = "locket" }, + { name = "toolz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } wheels = [ @@ -2603,7 +2603,7 @@ name = "patsy" version = "1.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/44/ed13eccdd0519eff265f44b670d46fbb0ec813e2274932dc1c0e48520f7d/patsy-1.0.2.tar.gz", hash = "sha256:cdc995455f6233e90e22de72c37fcadb344e7586fb83f06696f54d92f8ce74c0", size = 399942, upload-time = "2025-10-20T16:17:37.535Z" } wheels = [ @@ -2615,7 +2615,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "sys_platform == 'linux'" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -2668,11 +2668,11 @@ name = "pims" version = "0.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "imageio", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "slicerator", marker = "sys_platform == 'linux'" }, - { name = "tifffile", marker = "sys_platform == 'linux'" }, + { name = "imageio" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "slicerator" }, + { name = "tifffile" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/02/5bf3639f5b77e9b183011c08541c5039ba3d04f5316c70312b48a8e003a9/pims-0.7.tar.gz", hash = "sha256:55907a4c301256086d2aa4e34a5361b9109f24e375c2071e1117b9491e82946b", size = 87779, upload-time = "2024-06-10T19:20:42.842Z" } @@ -2708,7 +2708,7 @@ name = "prompt-toolkit" version = "3.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wcwidth", marker = "sys_platform == 'linux'" }, + { name = "wcwidth" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } wheels = [ @@ -2814,8 +2814,8 @@ name = "pybtex" version = "0.26.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "latexcodec", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "latexcodec" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/f5/f30da9c93f0fa6d619332b2f69597219b625f35780473a05164a9981fd9a/pybtex-0.26.1.tar.gz", hash = "sha256:2e5543bea424e60e9e42eef70bff597be48649d8f68ba061a7a092b2477d5464", size = 692991, upload-time = "2026-04-03T13:05:39.014Z" } wheels = [ @@ -2827,8 +2827,8 @@ name = "pybtex-docutils" version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "pybtex", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "pybtex" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/84/796ea94d26188a853660f81bded39f8de4cfe595130aef0dea1088705a11/pybtex-docutils-1.0.3.tar.gz", hash = "sha256:3a7ebdf92b593e00e8c1c538aa9a20bca5d92d84231124715acc964d51d93c6b", size = 18348, upload-time = "2023-08-22T18:47:54.833Z" } wheels = [ @@ -2840,10 +2840,10 @@ name = "pyccl" version = "3.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/57/40/ff357d61a20c21420f1fe7417ca6d889be2e07a67b3564775aeaf6c3f366/pyccl-3.3.4.tar.gz", hash = "sha256:a9d22557354fd161df909e8f4f199e4f1c434c080fd6cddb2246609ddbd70e7c", size = 16783719, upload-time = "2026-05-14T16:26:27.327Z" } wheels = [ @@ -2866,10 +2866,10 @@ name = "pydantic" version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-types", marker = "sys_platform == 'linux'" }, - { name = "pydantic-core", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, - { name = "typing-inspection", marker = "sys_platform == 'linux'" }, + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ @@ -2881,7 +2881,7 @@ name = "pydantic-core" version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ @@ -2934,7 +2934,7 @@ name = "pyerfa" version = "2.0.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/39/63cc8291b0cf324ae710df41527faf7d331bce573899199d926b3e492260/pyerfa-2.0.1.5.tar.gz", hash = "sha256:17d6b24fe4846c65d5e7d8c362dcb08199dc63b30a236aedd73875cc83e1f6c0", size = 818430, upload-time = "2024-11-11T15:22:30.852Z" } wheels = [ @@ -2957,8 +2957,8 @@ name = "pymaster" version = "2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ducc0", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, + { name = "ducc0" }, + { name = "healpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/95/45/470858f2a6a3f5133955bb87aed978a5e921d4052aea1c19718e6b72f0ae/pymaster-2.6.tar.gz", hash = "sha256:056a09bc8631cf338e93ef14d8c70272ad9554ef10498d919e920994531c57a2", size = 33707181, upload-time = "2025-12-16T09:18:28.905Z" } @@ -2967,8 +2967,8 @@ name = "pyopenssl" version = "26.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/74/b7/da07bae88f5a9506b4def6f2f4903cf4c3b8831e560dba8fa18ca08f758f/pyopenssl-26.3.0.tar.gz", hash = "sha256:589de7fae1c9ea670d18422ed00fc04da787bbde8e1454aea872aa57b49ad341", size = 182024, upload-time = "2026-06-12T20:28:07.458Z" } wheels = [ @@ -2989,10 +2989,10 @@ name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "iniconfig", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pluggy", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ @@ -3004,9 +3004,9 @@ name = "pytest-cov" version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coverage", marker = "sys_platform == 'linux'" }, - { name = "pluggy", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } wheels = [ @@ -3018,7 +3018,7 @@ name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, + { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ @@ -3076,7 +3076,7 @@ name = "pyzmq" version = "27.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy' and sys_platform == 'linux'" }, + { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } wheels = [ @@ -3105,10 +3105,10 @@ name = "qp-prob" version = "1.1.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecated", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "tables-io", marker = "sys_platform == 'linux'" }, + { name = "deprecated" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "tables-io" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b0/65/c54639ac63667800ee00d74c8851bd104ccaefcae95138073d6e77832d17/qp_prob-1.1.4.tar.gz", hash = "sha256:a68a3a3785d3ca57a2246c0524398338b7b7dffef0aef0722c231c64dde30107", size = 2460490, upload-time = "2026-06-25T20:08:34.63Z" } wheels = [ @@ -3120,9 +3120,9 @@ name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "sys_platform == 'linux'" }, - { name = "rpds-py", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -3134,8 +3134,8 @@ name = "regions" version = "0.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d3/86/44ff464efa2777247d6addc74416059f1872ab05c88588bbe1aadca27a7c/regions-0.12.tar.gz", hash = "sha256:1c9460770f250ef299e90a9d5c0b35941f7d05bbf879f6ffaa0538250c018ef9", size = 273609, upload-time = "2026-06-24T20:10:17.78Z" } wheels = [ @@ -3148,16 +3148,16 @@ name = "reproject" version = "0.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "astropy-healpix", marker = "sys_platform == 'linux'" }, - { name = "dask", extra = ["array"], marker = "sys_platform == 'linux'" }, - { name = "dask-image", marker = "sys_platform == 'linux'" }, - { name = "fsspec", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pillow", marker = "sys_platform == 'linux'" }, - { name = "pyavm", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "zarr", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "astropy-healpix" }, + { name = "dask", extra = ["array"] }, + { name = "dask-image" }, + { name = "fsspec" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "pyavm" }, + { name = "scipy" }, + { name = "zarr" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/44/6fd820ba336484277a91a2f4808b60d6ec0b9f033f588c237e778f20fe89/reproject-0.21.0.tar.gz", hash = "sha256:01ede715a1993c29431f52ff74189ef30f5e7b2e8b4dc88c1b002145a971dc1c", size = 1622661, upload-time = "2026-06-25T15:11:34.886Z" } wheels = [ @@ -3170,10 +3170,10 @@ name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "sys_platform == 'linux'" }, - { name = "charset-normalizer", marker = "sys_platform == 'linux'" }, - { name = "idna", marker = "sys_platform == 'linux'" }, - { name = "urllib3", marker = "sys_platform == 'linux'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -3185,7 +3185,7 @@ name = "rfc3339-validator" version = "0.1.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six", marker = "sys_platform == 'linux'" }, + { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } wheels = [ @@ -3206,7 +3206,7 @@ name = "rfc3987-syntax" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "lark", marker = "sys_platform == 'linux'" }, + { name = "lark" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz", hash = "sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d", size = 14239, upload-time = "2025-07-18T01:05:05.015Z" } wheels = [ @@ -3218,8 +3218,8 @@ name = "rich" version = "15.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, + { name = "markdown-it-py" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } wheels = [ @@ -3231,7 +3231,7 @@ name = "rocket-fft" version = "0.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numba", marker = "sys_platform == 'linux'" }, + { name = "numba" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/b0/09cbec3177ecf56e5ceb789a69fbcdcbabe758cbb682e5d36b14aa8c05fb/rocket_fft-0.3.1.tar.gz", hash = "sha256:ddc902a361099aff9fc02fe9ea3b0d036c2f67f93df356c2b754f7627d7fa4f4", size = 74276, upload-time = "2025-12-26T00:30:54.083Z" } wheels = [ @@ -3345,9 +3345,9 @@ name = "sacc" version = "2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "numpy" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b6/82/5b603d960279b8eb7fac677ba3550c0903a8390f1bff2eee6fbfa59765da/sacc-2.4.tar.gz", hash = "sha256:a98fb4e59b3c742ddb492a15a10e1f8a4279ba36d56c0d3c35e1cd13cca1167b", size = 424905, upload-time = "2026-07-02T11:29:27.458Z" } wheels = [ @@ -3359,7 +3359,7 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3390,9 +3390,9 @@ name = "seaborn" version = "0.13.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pandas" }, ] sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } wheels = [ @@ -3404,8 +3404,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -3435,29 +3435,29 @@ name = "shear-psf-leakage" version = "0.2.1" source = { git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop#0b3c17523b77760c9259be15ba5bbba99d575d0a" } dependencies = [ - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "cs-util", marker = "sys_platform == 'linux'" }, - { name = "emcee", marker = "sys_platform == 'linux'" }, - { name = "getdist", marker = "sys_platform == 'linux'" }, - { name = "gsl", marker = "sys_platform == 'linux'" }, - { name = "jupyter", marker = "sys_platform == 'linux'" }, - { name = "jupyter-server", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "jupytext", marker = "sys_platform == 'linux'" }, - { name = "lenspack", marker = "sys_platform == 'linux'" }, - { name = "lmfit", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "notebook", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "stats", marker = "sys_platform == 'linux'" }, - { name = "swig", marker = "sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, - { name = "tqdm", marker = "sys_platform == 'linux'" }, - { name = "treecorr", marker = "sys_platform == 'linux'" }, - { name = "uncertainties", marker = "sys_platform == 'linux'" }, + { name = "camb" }, + { name = "cs-util" }, + { name = "emcee" }, + { name = "getdist" }, + { name = "gsl" }, + { name = "jupyter" }, + { name = "jupyter-server" }, + { name = "jupyterlab" }, + { name = "jupytext" }, + { name = "lenspack" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "notebook" }, + { name = "pandas" }, + { name = "pyccl" }, + { name = "pyyaml" }, + { name = "scipy" }, + { name = "stats" }, + { name = "swig" }, + { name = "tornado" }, + { name = "tqdm" }, + { name = "treecorr" }, + { name = "uncertainties" }, ] [[package]] @@ -3474,11 +3474,11 @@ name = "skyproj" version = "2.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "hpgeom", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "healsparse" }, + { name = "hpgeom" }, + { name = "matplotlib" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/90/cf/5158151ae6fc60459f430451b612a4bf2da07ebff2a9f52a37e73715a5ee/skyproj-2.5.0.tar.gz", hash = "sha256:cb8d5115927ca43cacdb6d92f00bb4fcc8563ee71028f755203ce1752e67c140", size = 8521587, upload-time = "2026-06-26T19:51:15.378Z" } wheels = [ @@ -3500,7 +3500,7 @@ name = "smart-open" version = "7.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt", marker = "sys_platform == 'linux'" }, + { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/c6/22e7a2acd5d27941e85e0d7ede398da5abe2e4677d2265c924157247c32e/smart_open-7.7.1.tar.gz", hash = "sha256:9414ba5733e28309f29b28a303b0f1054ad23fe0275f1a1b600c80a724f4bd1a", size = 54952, upload-time = "2026-06-26T07:56:35.309Z" } wheels = [ @@ -3519,15 +3519,16 @@ wheels = [ [[package]] name = "smokescreen" version = "1.5.6" -source = { git = "https://github.com/UNIONS-WL/Smokescreen?rev=588a6b9b26560bd5ba3dd5ba342f3c40152644f9#588a6b9b26560bd5ba3dd5ba342f3c40152644f9" } +source = { git = "https://github.com/UNIONS-WL/Smokescreen?rev=main#b457c995dba8421bd7804fe8324ba8c67cc56a43" } dependencies = [ - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "jsonargparse", extra = ["signatures"], marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "sacc", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "astropy" }, + { name = "cryptography" }, + { name = "h5py" }, + { name = "jsonargparse", extra = ["signatures"] }, + { name = "numpy" }, + { name = "pyccl" }, + { name = "sacc" }, + { name = "scipy" }, ] [[package]] @@ -3535,37 +3536,37 @@ name = "snakemake" version = "9.23.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "conda-inject", marker = "sys_platform == 'linux'" }, - { name = "configargparse", marker = "sys_platform == 'linux'" }, - { name = "connection-pool", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "dpath", marker = "sys_platform == 'linux'" }, - { name = "gitpython", marker = "sys_platform == 'linux'" }, - { name = "humanfriendly", marker = "sys_platform == 'linux'" }, - { name = "immutables", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "jsonschema", marker = "sys_platform == 'linux'" }, - { name = "nbformat", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "platformdirs", marker = "sys_platform == 'linux'" }, - { name = "psutil", marker = "sys_platform == 'linux'" }, - { name = "pulp", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "referencing", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "smart-open", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-executor-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-logger-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-report-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-scheduler-plugins", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-storage-plugins", marker = "sys_platform == 'linux'" }, - { name = "sqlmodel", marker = "sys_platform == 'linux'" }, - { name = "tabulate", marker = "sys_platform == 'linux'" }, - { name = "tenacity", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, - { name = "wrapt", marker = "sys_platform == 'linux'" }, - { name = "yte", marker = "sys_platform == 'linux'" }, + { name = "conda-inject" }, + { name = "configargparse" }, + { name = "connection-pool" }, + { name = "docutils" }, + { name = "dpath" }, + { name = "gitpython" }, + { name = "humanfriendly" }, + { name = "immutables" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "psutil" }, + { name = "pulp" }, + { name = "pyyaml" }, + { name = "referencing" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "snakemake-interface-common" }, + { name = "snakemake-interface-executor-plugins" }, + { name = "snakemake-interface-logger-plugins" }, + { name = "snakemake-interface-report-plugins" }, + { name = "snakemake-interface-scheduler-plugins" }, + { name = "snakemake-interface-storage-plugins" }, + { name = "sqlmodel" }, + { name = "tabulate" }, + { name = "tenacity" }, + { name = "throttler" }, + { name = "wrapt" }, + { name = "yte" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/c2/45aa858e55edbb7dbc243ed60859f4f3d92d63ecdd67a80632aa3659fe6b/snakemake-9.23.1.tar.gz", hash = "sha256:ef8d698bfce66a6669cc29df7e344b0b367fd90956c725e5ae0a79a556e8e93f", size = 6802587, upload-time = "2026-06-18T09:42:57.653Z" } wheels = [ @@ -3577,9 +3578,9 @@ name = "snakemake-interface-common" version = "1.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "configargparse", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "argparse-dataclass" }, + { name = "configargparse" }, + { name = "packaging" }, ] sdist = { url = "https://files.pythonhosted.org/packages/89/c3/592f832f6e5d2d31f749392e48e8401b7625dec668d3d365d8d28f2b6c30/snakemake_interface_common-1.23.0.tar.gz", hash = "sha256:6ed14531a461417659364a0dd0acc51b786af4e26fc15cc5e00ff3d9fcaffacc", size = 13960, upload-time = "2026-03-08T21:54:29.251Z" } wheels = [ @@ -3591,9 +3592,9 @@ name = "snakemake-interface-executor-plugins" version = "9.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, + { name = "argparse-dataclass" }, + { name = "snakemake-interface-common" }, + { name = "throttler" }, ] sdist = { url = "https://files.pythonhosted.org/packages/54/50/de06b284c45a8e94fb8e4a12d5235065e78b49b8f84329dc10fe39f4b7dd/snakemake_interface_executor_plugins-9.4.0.tar.gz", hash = "sha256:9d4138897beacbaadaedad94b63f948eaeb604b7fc78f9cf65ac57f090f2c066", size = 16549, upload-time = "2026-03-08T17:04:02.644Z" } wheels = [ @@ -3605,7 +3606,7 @@ name = "snakemake-interface-logger-plugins" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/0c/3fa5d592663c65669a867526604aadc6fbc235fb9284e94b49c0ef59aa41/snakemake_interface_logger_plugins-2.1.0.tar.gz", hash = "sha256:c89a00d2a398490cecd91b6dc6db8049cba93712d82e1d8f3000f3040bf3791c", size = 15917, upload-time = "2026-05-20T15:12:35.259Z" } wheels = [ @@ -3617,7 +3618,7 @@ name = "snakemake-interface-report-plugins" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/d6/6160ed98de665d6871dd356597dbf726688cc786e88668359ca37b7d9f54/snakemake_interface_report_plugins-1.3.0.tar.gz", hash = "sha256:fc9495298bec4e69721ab8afe6d6d88a86966fda2eeb003db56b9a88b86d5934", size = 4283, upload-time = "2025-10-31T10:52:36.55Z" } wheels = [ @@ -3629,7 +3630,7 @@ name = "snakemake-interface-scheduler-plugins" version = "2.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, + { name = "snakemake-interface-common" }, ] sdist = { url = "https://files.pythonhosted.org/packages/88/d9/d480807d2cfc2d132bc760d877d45ec8fbe620a24200ec4d2697c4a26031/snakemake_interface_scheduler_plugins-2.0.2.tar.gz", hash = "sha256:2797e8fa9019d983132c2b403f14d6fcd3c5ad4c8d8a66b984b4740a71cacc46", size = 8642, upload-time = "2025-10-20T13:58:12.988Z" } wheels = [ @@ -3641,11 +3642,11 @@ name = "snakemake-interface-storage-plugins" version = "4.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly", marker = "sys_platform == 'linux'" }, - { name = "snakemake-interface-common", marker = "sys_platform == 'linux'" }, - { name = "tenacity", marker = "sys_platform == 'linux'" }, - { name = "throttler", marker = "sys_platform == 'linux'" }, - { name = "wrapt", marker = "sys_platform == 'linux'" }, + { name = "humanfriendly" }, + { name = "snakemake-interface-common" }, + { name = "tenacity" }, + { name = "throttler" }, + { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/6e/f3c5b2d621fd6a6b78d8cfc01fef6b926fe2c277f5ed77c5e4deeacb94eb/snakemake_interface_storage_plugins-4.4.1.tar.gz", hash = "sha256:b2b5bf05318af36955ebf2ce76c921c0fb06904ca98fb30e1657d88b7b7b6945", size = 14924, upload-time = "2026-03-16T11:16:01.075Z" } wheels = [ @@ -3675,81 +3676,81 @@ name = "sp-validation" version = "0.6.0" source = { virtual = "." } dependencies = [ - { name = "adjusttext", marker = "sys_platform == 'linux'" }, - { name = "astropy", marker = "sys_platform == 'linux'" }, - { name = "camb", marker = "sys_platform == 'linux'" }, - { name = "clmm", marker = "sys_platform == 'linux'" }, - { name = "colorama", marker = "sys_platform == 'linux'" }, - { name = "cosmo-numba", marker = "sys_platform == 'linux'" }, - { name = "cryptography", marker = "sys_platform == 'linux'" }, - { name = "cs-util", marker = "sys_platform == 'linux'" }, - { name = "emcee", marker = "sys_platform == 'linux'" }, - { name = "getdist", marker = "sys_platform == 'linux'" }, - { name = "h5py", marker = "sys_platform == 'linux'" }, - { name = "healpy", marker = "sys_platform == 'linux'" }, - { name = "healsparse", marker = "sys_platform == 'linux'" }, - { name = "importlib-metadata", marker = "sys_platform == 'linux'" }, - { name = "joblib", marker = "sys_platform == 'linux'" }, - { name = "jupyter", marker = "sys_platform == 'linux'" }, - { name = "jupyterlab", marker = "sys_platform == 'linux'" }, - { name = "jupytext", marker = "sys_platform == 'linux'" }, - { name = "lenspack", marker = "sys_platform == 'linux'" }, - { name = "lmfit", marker = "sys_platform == 'linux'" }, - { name = "matplotlib", marker = "sys_platform == 'linux'" }, - { name = "numba", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "opencv-python-headless", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "pyarrow", marker = "sys_platform == 'linux'" }, - { name = "pyccl", marker = "sys_platform == 'linux'" }, - { name = "pymaster", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "regions", marker = "sys_platform == 'linux'" }, - { name = "reproject", marker = "sys_platform == 'linux'" }, - { name = "sacc", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, - { name = "seaborn", marker = "sys_platform == 'linux'" }, - { name = "shear-psf-leakage", marker = "sys_platform == 'linux'" }, - { name = "skyproj", marker = "sys_platform == 'linux'" }, - { name = "smokescreen", marker = "sys_platform == 'linux'" }, - { name = "statsmodels", marker = "sys_platform == 'linux'" }, - { name = "tqdm", marker = "sys_platform == 'linux'" }, - { name = "treecorr", marker = "sys_platform == 'linux'" }, - { name = "uncertainties", marker = "sys_platform == 'linux'" }, + { name = "adjusttext" }, + { name = "astropy" }, + { name = "camb" }, + { name = "clmm" }, + { name = "colorama" }, + { name = "cosmo-numba" }, + { name = "cryptography" }, + { name = "cs-util" }, + { name = "emcee" }, + { name = "getdist" }, + { name = "h5py" }, + { name = "healpy" }, + { name = "healsparse" }, + { name = "importlib-metadata" }, + { name = "joblib" }, + { name = "jupyter" }, + { name = "jupyterlab" }, + { name = "jupytext" }, + { name = "lenspack" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "numba" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyccl" }, + { name = "pymaster" }, + { name = "pyyaml" }, + { name = "regions" }, + { name = "reproject" }, + { name = "sacc" }, + { name = "scipy" }, + { name = "seaborn" }, + { name = "shear-psf-leakage" }, + { name = "skyproj" }, + { name = "smokescreen" }, + { name = "statsmodels" }, + { name = "tqdm" }, + { name = "treecorr" }, + { name = "uncertainties" }, ] [package.optional-dependencies] develop = [ - { name = "myst-parser", marker = "sys_platform == 'linux'" }, - { name = "numpydoc", marker = "sys_platform == 'linux'" }, - { name = "pytest", marker = "sys_platform == 'linux'" }, - { name = "pytest-cov", marker = "sys_platform == 'linux'" }, - { name = "ruff", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, - { name = "sphinxawesome-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, + { name = "myst-parser" }, + { name = "numpydoc" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, + { name = "sphinx" }, + { name = "sphinxawesome-theme" }, + { name = "sphinxcontrib-bibtex" }, ] docs = [ - { name = "myst-parser", marker = "sys_platform == 'linux'" }, - { name = "numpydoc", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, - { name = "sphinxawesome-theme", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-bibtex", marker = "sys_platform == 'linux'" }, + { name = "myst-parser" }, + { name = "numpydoc" }, + { name = "sphinx" }, + { name = "sphinxawesome-theme" }, + { name = "sphinxcontrib-bibtex" }, ] glass = [ - { name = "cosmology", marker = "sys_platform == 'linux'" }, - { name = "fitsio", marker = "sys_platform == 'linux'" }, - { name = "glass", marker = "sys_platform == 'linux'" }, - { name = "glass-ext-camb", marker = "sys_platform == 'linux'" }, + { name = "cosmology" }, + { name = "fitsio" }, + { name = "glass" }, + { name = "glass-ext-camb" }, ] test = [ - { name = "pytest", marker = "sys_platform == 'linux'" }, - { name = "pytest-cov", marker = "sys_platform == 'linux'" }, - { name = "ruff", marker = "sys_platform == 'linux'" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, ] workflow = [ - { name = "mpi4py", marker = "sys_platform == 'linux'" }, - { name = "snakemake", marker = "sys_platform == 'linux'" }, + { name = "mpi4py" }, + { name = "snakemake" }, ] [package.metadata] @@ -3800,7 +3801,7 @@ requires-dist = [ { name = "seaborn" }, { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, { name = "skyproj" }, - { name = "smokescreen", git = "https://github.com/UNIONS-WL/Smokescreen?rev=588a6b9b26560bd5ba3dd5ba342f3c40152644f9" }, + { name = "smokescreen", git = "https://github.com/UNIONS-WL/Smokescreen?rev=main" }, { name = "snakemake", marker = "extra == 'workflow'" }, { name = "sp-validation", extras = ["test", "docs"], marker = "extra == 'develop'" }, { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0" }, @@ -3818,22 +3819,22 @@ name = "sphinx" version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "alabaster", marker = "sys_platform == 'linux'" }, - { name = "babel", marker = "sys_platform == 'linux'" }, - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "imagesize", marker = "sys_platform == 'linux'" }, - { name = "jinja2", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pygments", marker = "sys_platform == 'linux'" }, - { name = "requests", marker = "sys_platform == 'linux'" }, - { name = "roman-numerals", marker = "sys_platform == 'linux'" }, - { name = "snowballstemmer", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-applehelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-devhelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-htmlhelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-jsmath", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-qthelp", marker = "sys_platform == 'linux'" }, - { name = "sphinxcontrib-serializinghtml", marker = "sys_platform == 'linux'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -3845,8 +3846,8 @@ name = "sphinxawesome-theme" version = "6.0.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "beautifulsoup4", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "beautifulsoup4" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9a/3f/ba7f8dc2837c0ac362127c6cd491578c23ac4d4e8cc6a37726531be29256/sphinxawesome_theme-6.0.5.tar.gz", hash = "sha256:ed3d82b7f0e30e3d6f38f5245d89775e9472a80a45ffd73d6a0a4b9d05ac5736", size = 343775, upload-time = "2026-06-15T15:09:36.094Z" } wheels = [ @@ -3867,10 +3868,10 @@ name = "sphinxcontrib-bibtex" version = "2.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docutils", marker = "sys_platform == 'linux'" }, - { name = "pybtex", marker = "sys_platform == 'linux'" }, - { name = "pybtex-docutils", marker = "sys_platform == 'linux'" }, - { name = "sphinx", marker = "sys_platform == 'linux'" }, + { name = "docutils" }, + { name = "pybtex" }, + { name = "pybtex-docutils" }, + { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/6a/8e0b2c2420286389e7fed78ff361ec30e2f1d58c8560af8d64df5e7b61e0/sphinxcontrib_bibtex-2.7.0.tar.gz", hash = "sha256:fee700f7aae29bb8f654c62913f00d34ac44fc0b8ca0fa67ac922ff4453addee", size = 120669, upload-time = "2026-05-06T09:29:24.935Z" } wheels = [ @@ -3927,8 +3928,8 @@ name = "sqlalchemy" version = "2.0.51" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "greenlet", marker = "(platform_machine == 'AMD64' and sys_platform == 'linux') or (platform_machine == 'WIN32' and sys_platform == 'linux') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'amd64' and sys_platform == 'linux') or (platform_machine == 'ppc64le' and sys_platform == 'linux') or (platform_machine == 'win32' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } wheels = [ @@ -3956,8 +3957,8 @@ name = "sqlmodel" version = "0.0.37" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic", marker = "sys_platform == 'linux'" }, - { name = "sqlalchemy", marker = "sys_platform == 'linux'" }, + { name = "pydantic" }, + { name = "sqlalchemy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fb/26/1d2faa0fd5a765267f49751de533adac6b9ff9366c7c6e7692df4f32230f/sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352", size = 85527, upload-time = "2026-02-21T16:39:47.038Z" } wheels = [ @@ -3969,9 +3970,9 @@ name = "stack-data" version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "asttokens", marker = "sys_platform == 'linux'" }, - { name = "executing", marker = "sys_platform == 'linux'" }, - { name = "pure-eval", marker = "sys_platform == 'linux'" }, + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } wheels = [ @@ -3989,11 +3990,11 @@ name = "statsmodels" version = "0.14.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "patsy", marker = "sys_platform == 'linux'" }, - { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "patsy" }, + { name = "scipy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a", size = 20689085, upload-time = "2025-12-05T23:08:39.522Z" } wheels = [ @@ -4035,7 +4036,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath", marker = "sys_platform == 'linux'" }, + { name = "mpmath" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -4047,11 +4048,11 @@ name = "tables" version = "3.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "blosc2", marker = "sys_platform == 'linux'" }, - { name = "numexpr", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "py-cpuinfo", marker = "sys_platform == 'linux'" }, + { name = "blosc2" }, + { name = "numexpr" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "py-cpuinfo" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/a3/d213ebe7376d48055bd55a29cd9f99061afa0dcece608f94a5025d797b0a/tables-3.11.1.tar.gz", hash = "sha256:78abcf413091bc7c1e4e8c10fbbb438d1ac0b5a87436c5b972c3e8253871b6fb", size = 4790533, upload-time = "2026-03-01T11:43:36.036Z" } wheels = [ @@ -4066,12 +4067,12 @@ name = "tables-io" version = "1.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "sys_platform == 'linux'" }, - { name = "deprecated", marker = "sys_platform == 'linux'" }, - { name = "h5py", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pandas", marker = "sys_platform == 'linux'" }, - { name = "tables", marker = "sys_platform == 'linux'" }, + { name = "click" }, + { name = "deprecated" }, + { name = "h5py" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "tables" }, ] sdist = { url = "https://files.pythonhosted.org/packages/12/1b/61bed3c55ce4602be5379362ce939b71a2e707e856ee497c1d1e72f8ff3d/tables_io-1.1.2.tar.gz", hash = "sha256:ea27eac8b22ff4e3500afd92140794bd41161f768a2c84520b91b3b58e945182", size = 91394, upload-time = "2026-06-18T16:21:20.896Z" } wheels = [ @@ -4101,8 +4102,8 @@ name = "terminado" version = "0.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "os_name != 'nt' and sys_platform == 'linux'" }, - { name = "tornado", marker = "sys_platform == 'linux'" }, + { name = "ptyprocess", marker = "os_name != 'nt'" }, + { name = "tornado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } wheels = [ @@ -4132,7 +4133,7 @@ name = "tifffile" version = "2026.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/38/5e2ecef5af2f4fd4a89bb8d6240de9458bab4d51a4cbd97aeb3a0cd618e2/tifffile-2026.6.1.tar.gz", hash = "sha256:626c892c0e899d959b9438e7c0e1491dc154a7fead1f1f37a991724a50eceba9", size = 429694, upload-time = "2026-05-31T23:57:12.165Z" } wheels = [ @@ -4144,7 +4145,7 @@ name = "tinycss2" version = "1.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "webencodings", marker = "sys_platform == 'linux'" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/ae/2ca4913e5c0f09781d75482874c3a95db9105462a92ddd303c7d285d3df2/tinycss2-1.5.1.tar.gz", hash = "sha256:d339d2b616ba90ccce58da8495a78f46e55d4d25f9fd71dfd526f07e7d53f957", size = 88195, upload-time = "2025-11-23T10:29:10.082Z" } wheels = [ @@ -4195,8 +4196,8 @@ name = "transformcl" version = "2026.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "array-api-compat", marker = "sys_platform == 'linux'" }, - { name = "flt", marker = "sys_platform == 'linux'" }, + { name = "array-api-compat" }, + { name = "flt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/ce/ae8373b67259c3ea6033425161f7d195387b1e68b17d4d00eee1b68e5edd/transformcl-2026.1.tar.gz", hash = "sha256:14551c33d164f3db24b9141f1f77553182a4b9c789f883476eb33d87ea4d8f13", size = 4850, upload-time = "2026-03-02T19:18:21.221Z" } wheels = [ @@ -4208,11 +4209,11 @@ name = "treecorr" version = "5.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "lsstdesc-coord", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "pybind11", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, - { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "lsstdesc-coord" }, + { name = "numpy" }, + { name = "pybind11" }, + { name = "pyyaml" }, + { name = "setuptools" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/95/ff399cb37b9e3d7c758f8cf0d5757f06a38093f223f2a65d7cc5b50313d5/treecorr-5.1.3.tar.gz", hash = "sha256:d921ccace064325a8ebb93f371c624f13fcc9559373e3b37cc216cb94ae81ed2", size = 1146147, upload-time = "2026-03-18T18:38:03.626Z" } wheels = [ @@ -4227,8 +4228,8 @@ name = "typeshed-client" version = "2.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "importlib-resources", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "importlib-resources" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/54/5d/97d6fa8c204b0d42be931e7a568b306cda43f93ff45f8b47537ee61390de/typeshed_client-2.12.0.tar.gz", hash = "sha256:54dcfa25fcff82cedcb1b51c03c1b3c51a614097aab8fe49e4316aff6f79d9c7", size = 529457, upload-time = "2026-06-02T04:00:51.51Z" } wheels = [ @@ -4249,7 +4250,7 @@ name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ @@ -4297,9 +4298,9 @@ name = "vos" version = "3.6.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aenum", marker = "sys_platform == 'linux'" }, - { name = "cadcutils", marker = "sys_platform == 'linux'" }, - { name = "html2text", marker = "sys_platform == 'linux'" }, + { name = "aenum" }, + { name = "cadcutils" }, + { name = "html2text" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3e/6c/445bbedf2f90e71c09ff890aa8b409af5d074b1578eef383117cbcdfd09a/vos-3.6.4.tar.gz", hash = "sha256:67e4e24689ee881124b6670d7d2fc6527cc404687fc527fffc20cf5b1bd4c0bd", size = 103445, upload-time = "2025-12-12T00:20:56.398Z" } wheels = [ @@ -4395,9 +4396,9 @@ name = "yte" version = "1.9.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "argparse-dataclass", marker = "sys_platform == 'linux'" }, - { name = "dpath", marker = "sys_platform == 'linux'" }, - { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "argparse-dataclass" }, + { name = "dpath" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/f5/7e44620e6e077bfe624b9a17c329b8e0d0159e176e1f1a93c2790428ab2c/yte-1.9.4.tar.gz", hash = "sha256:86a47e6d722cec9419a7ac88be57d0d6c4ce28f02860393b71a66f2c674069f6", size = 8101, upload-time = "2025-11-27T12:55:00.85Z" } wheels = [ @@ -4409,12 +4410,12 @@ name = "zarr" version = "3.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "donfig", marker = "sys_platform == 'linux'" }, - { name = "google-crc32c", marker = "sys_platform == 'linux'" }, - { name = "numcodecs", marker = "sys_platform == 'linux'" }, - { name = "numpy", marker = "sys_platform == 'linux'" }, - { name = "packaging", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "donfig" }, + { name = "google-crc32c" }, + { name = "numcodecs" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/8d/aeb164004f87543b06ef54f885d02c342c31ceb274e2bbec470a98927621/zarr-3.2.1.tar.gz", hash = "sha256:71565b738a0e7e8ed226f0516eba8c6bb53440ad7669a8c48ebb3534a161d035", size = 675161, upload-time = "2026-05-05T12:37:22.383Z" } wheels = [ From fdd78f0fc95905189fc70188da40920a75d27835 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 31 Aug 2026 03:40:56 +0200 Subject: [PATCH 04/83] Stop tracking CI lint-gate scratch files check.txt/format.txt/report.md are written by lint.yml during each run; the autofix step's 'git add -A' kept committing them back. Gitignore them (plus the residual-pass variants) and drop the tracked copies. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012b34pAS3bXRxN5hVdyq5Sw --- .gitignore | 9 ++++++++- check.txt | 1 - format.txt | 2 -- report.md | 10 ---------- 4 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 check.txt delete mode 100644 format.txt delete mode 100644 report.md diff --git a/.gitignore b/.gitignore index 82fb2697..a1a0394b 100644 --- a/.gitignore +++ b/.gitignore @@ -194,4 +194,11 @@ papers/catalog/plots/*.pdf papers/cosmo_val/logs/ # Ignore scratch notebooks -scratch/*/*.ipynb \ No newline at end of file +scratch/*/*.ipynb +# CI lint-gate scratch output (lint.yml writes these in the run tree; never commit) +check.txt +format.txt +report.md +check2.txt +format2.txt +residual.md diff --git a/check.txt b/check.txt deleted file mode 100644 index 1f5f344d..00000000 --- a/check.txt +++ /dev/null @@ -1 +0,0 @@ -All checks passed! diff --git a/format.txt b/format.txt deleted file mode 100644 index a0b12d74..00000000 --- a/format.txt +++ /dev/null @@ -1,2 +0,0 @@ -Would reformat: src/sp_validation/tests/test_masks.py -1 file would be reformatted, 225 files already formatted diff --git a/report.md b/report.md deleted file mode 100644 index e681a67c..00000000 --- a/report.md +++ /dev/null @@ -1,10 +0,0 @@ -### `ruff check .` - -✅ clean - -### `ruff format --check .` - -``` -Would reformat: src/sp_validation/tests/test_masks.py -1 file would be reformatted, 225 files already formatted -``` From d9c989681bbb172f480100641f700ab7b3cadec7 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Sat, 5 Sep 2026 16:14:12 +0200 Subject: [PATCH 05/83] PhotoPipe + ShapePipe (#310) * additive bias calculation for paper updated to v1.4.6.3 * added config * added fill_photoz script * fill photoz bands fixes * adding mag errors to fill_photoz * added Z_ML to fill_photoz * added more flags to fill_photoz * added 0p7 and 1p0 aperture magnitudes to PhotoPipe + SP output * Fixed fill photoz script with new MP_NAME type * ruff autofix (format + safe lint fixes) Pushed by the lint gate. * removed leftover git marker * fill_photoz_bands: spot-check FITS/HDF5 row order before filling a tile The fill pairs FITS row k with the k-th HDF5 row of the tile (sorted-index order) and only ever verified the row *count*, so a PhotoPipe tile ordered differently from the comprehensive catalogue would be filled with silently mismatched photo-z. After the size check, compare RA/Dec for a small sample of rows -- up to five at each end plus evenly spaced interior rows, --n_check_rows (default 10), --check_tol_arcsec (default 0.5). Only the sampled HDF5 rows are read (one fancy-index into the already-sorted index array), so the cost is a handful of point reads per tile rather than a full per-row match. A failing tile is warned about, counted as a row-order mismatch in the end-of-run summary, counted towards the consecutive-failure abort, and skipped without being added to done_tiles -- exactly as a size mismatch is, so a resume retries it. --n_check_rows 0 disables the check. HDF5 columns are RA/Dec (cat_config.yaml ra_col/dec_col for SP_v1.4.x). The FITS names are resolved at runtime from a candidate list; ALPHA_J2000 / DELTA_J2000 is first, verified against a real DR6 tile (/n17data/UNIONS/WL/photometry/UNIONS_DR6/UNIONS.001.227_SP_ugriz_photoz_ext.cat). If no candidate pair matches, the check disables itself with a warning rather than skipping tiles. Test: synthetic 3-tile HDF5 + FITS pair, tiles interleaved so the non-contiguous write path runs too; the two aligned tiles fill, the tile with reversed FITS rows is skipped and left empty, and --n_check_rows 0 reproduces the old unchecked behaviour. * fill_photoz_bands: row-order check ignores invalid positions, fails if none A sampled row with a non-finite or sentinel (|Dec| > 90) coordinate on either side carries no information about the FITS/HDF5 pairing. Before, such a row made `sep <= tol` False (a false row-order failure), and an all-NaN sample raised a RuntimeWarning from np.nanmax. Now those rows are excluded from the comparison, n_checked reports only the rows actually compared, and a tile whose sample has no comparable row is treated as unverifiable and skipped (distinct warning), so it is retried on resume rather than written blind. Adds a direct unit test covering partial/all-invalid samples, a reversed remainder, and a single-row tile (one-element h5py fancy index). Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: martinkilbinger Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Cail Daley Co-authored-by: Claude Fable 5.1 --- cosmo_val/cat_config.yaml | 108 ++- scripts/check_filled_fields.py | 74 ++ scripts/fill_photoz_bands.py | 874 ++++++++++++++++++ .../tests/test_fill_photoz_bands.py | 304 ++++++ 4 files changed, 1353 insertions(+), 7 deletions(-) create mode 100644 scripts/check_filled_fields.py create mode 100755 scripts/fill_photoz_bands.py create mode 100644 src/sp_validation/tests/test_fill_photoz_bands.py diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index cc1326df..cae6411b 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -980,7 +980,98 @@ SP_v1.4.11.3: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.12.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: lightblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: ^ + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.13.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: h + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 @@ -1038,11 +1129,12 @@ SP_v1.4.11.3_ecut07: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.6_uncal: +SP_v1.4.6.3_uncal: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1053,11 +1145,12 @@ SP_v1.4.6_uncal: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 -SP_v1.4.6_uncal_w_iv: +SP_v1.4.6.3_uncal_w_iv: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1068,11 +1161,12 @@ SP_v1.4.6_uncal_w_iv: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 -SP_v1.4.6_uncal_w_1: +SP_v1.4.6.3_uncal_w_1: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal diff --git a/scripts/check_filled_fields.py b/scripts/check_filled_fields.py new file mode 100644 index 00000000..675ad116 --- /dev/null +++ b/scripts/check_filled_fields.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Check how many entries in selected HDF5 fields are filled (value != -199). + +Reads in chunks with a progress bar and periodic fill-fraction updates. +""" + +import h5py +import numpy as np +from tqdm import tqdm + +HDF5_FILE = "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.6.c.DR6.hdf5" +EMPTY_VALUE = -199 +CHUNK_SIZE = 5_000_000 # rows per chunk +REPORT_EVERY = 10 # print running fractions every N chunks + +FIELDS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "MAG_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAG_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAG_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAG_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAG_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAG_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", +] + +with h5py.File(HDF5_FILE, "r") as f: + data = f["data"] + n_total = data.shape[0] + n_chunks = (n_total + CHUNK_SIZE - 1) // CHUNK_SIZE + print(f"Total entries : {n_total:,}") + print(f"Chunk size : {CHUNK_SIZE:,} ({n_chunks} chunks)\n") + + counts = {field: 0 for field in FIELDS} + + with tqdm(total=n_total, unit="rows", unit_scale=True, desc="Reading") as pbar: + for chunk_idx in range(n_chunks): + start = chunk_idx * CHUNK_SIZE + end = min(start + CHUNK_SIZE, n_total) + + for field in FIELDS: + counts[field] += int(np.sum(data[field, start:end] != EMPTY_VALUE)) + + pbar.update(end - start) + + # Periodic running-fraction report + if (chunk_idx + 1) % REPORT_EVERY == 0 or (chunk_idx + 1) == n_chunks: + rows_done = end + tqdm.write( + f"\n --- after {rows_done:,} rows ({100 * rows_done / n_total:.1f}%) ---" + ) + tqdm.write(f" {'Field':<22} {'Filled %':>9}") + for field in FIELDS: + pct = 100.0 * counts[field] / rows_done + tqdm.write(f" {field:<22} {pct:>8.2f}%") + +# Final summary +print(f"\n{'=' * 56}") +print(f"FINAL SUMMARY (total rows: {n_total:,})") +print(f"{'Field':<22} {'Filled':>12} {'Empty':>12} {'Filled %':>10}") +print("-" * 60) +for field in FIELDS: + n_filled = counts[field] + n_empty = n_total - n_filled + pct = 100.0 * n_filled / n_total + print(f"{field:<22} {n_filled:>12,} {n_empty:>12,} {pct:>9.2f}%") diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py new file mode 100755 index 00000000..36a0a1b2 --- /dev/null +++ b/scripts/fill_photoz_bands.py @@ -0,0 +1,874 @@ +#!/usr/bin/env python + +"""fill_photoz_bands.py + +Add PhotoPipe photo-z and multi-band magnitude fields to a ShapePipe +comprehensive HDF5 catalogue and fill them from PhotoPipe FITS tiles. + +Two phases, both resumable via a checkpoint file: + + Phase 1 — Create output: copy the input HDF5 and append the new + columns initialised to EMPTY_VALUE (-199). + Phase 2 — Fill tiles: for each PhotoPipe FITS tile found in fits_dir, + write the field values into the output file. + +The fill assumes that, within a tile, the PhotoPipe FITS rows are in the +same order as the HDF5 rows of that tile. Before writing, this is checked +on a small sample of rows by comparing their sky positions (see +``check_tile_row_order``); tiles that fail are skipped, not written. + +Skips missing FITS files. Warns (does not error) on missing PhotoPipe +keys. Supports interrupt + restart at any point. + +:Authors: Martin Kilbinger + +""" + +import json +import os +import sys +import warnings +from collections import defaultdict +from timeit import default_timer as timer + +import h5py +import numpy as np +import tqdm +from astropy.io import fits +from cs_util import args as cs_args +from cs_util import logging + +FITS_HDU = 1 +EMPTY_VALUE = -199 +COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output +SCAN_CHUNK = 5_000_000 # rows per chunk when scanning TILE_ID +MAX_CONSEC_FAILS = 10 # abort if this many tiles in a row fail + +# Sky-position columns of the ShapePipe comprehensive HDF5 catalogue +# (cosmo_val/cat_config.yaml: ra_col: RA, dec_col: Dec for all SP_v1.4.x). +RA_COL_HDF5 = "RA" +DEC_COL_HDF5 = "Dec" + +# Candidate (RA, Dec) column names in the PhotoPipe FITS tiles, most likely +# first. DR6 tiles (UNIONS.*_SP_ugriz_photoz_ext.cat) carry ALPHA_J2000 / +# DELTA_J2000; the remaining pairs are fallbacks for other PhotoPipe outputs. +FITS_RADEC_CANDIDATES = [ + ("ALPHA_J2000", "DELTA_J2000"), + ("RA", "DEC"), + ("RA", "Dec"), + ("X_WORLD", "Y_WORLD"), +] + +REQUESTED_KEYS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "Z_ML", + "MAG_GAAP_u", + "MAGERR_GAAP_u", + "MAG_GAAP_0p7_u", + "MAGERR_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAGERR_GAAP_1p0_u", + "FLAG_GAAP_u", + "MAG_LIM_u", + "FLUX_GAAP_u", + "FLUXERR_GAAP_u", + "EXTINCTION_u", + "MAG_GAAP_g", + "MAGERR_GAAP_g", + "MAG_GAAP_0p7_g", + "MAGERR_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAGERR_GAAP_1p0_g", + "FLAG_GAAP_g", + "MAG_LIM_g", + "FLUX_GAAP_g", + "FLUXERR_GAAP_g", + "EXTINCTION_g", + "MAG_GAAP_r", + "MAGERR_GAAP_r", + "MAG_GAAP_0p7_r", + "MAGERR_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAGERR_GAAP_1p0_r", + "FLAG_GAAP_r", + "MAG_LIM_r", + "FLUX_GAAP_r", + "FLUXERR_GAAP_r", + "EXTINCTION_r", + "MAG_GAAP_i", + "MAGERR_GAAP_i", + "MAG_GAAP_0p7_i", + "MAGERR_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAGERR_GAAP_1p0_i", + "FLAG_GAAP_i", + "MAG_LIM_i", + "FLUX_GAAP_i", + "FLUXERR_GAAP_i", + "EXTINCTION_i", + "MAG_GAAP_z", + "MAGERR_GAAP_z", + "MAG_GAAP_0p7_z", + "MAGERR_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAGERR_GAAP_1p0_z", + "FLAG_GAAP_z", + "MAG_LIM_z", + "FLUX_GAAP_z", + "FLUXERR_GAAP_z", + "EXTINCTION_z", + "MAG_GAAP_z2", + "MAGERR_GAAP_z2", + "MAG_GAAP_0p7_z2", + "MAGERR_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", + "MAGERR_GAAP_1p0_z2", + "FLAG_GAAP_z2", + "MAG_LIM_z2", + "FLUX_GAAP_z2", + "FLUXERR_GAAP_z2", + "EXTINCTION_z2", + "EXTINCTION", + "ODDS", + "CHI_SQUARED_BPZ", + "M_0", + "BPZ_FILT", + "BPZ_NONDETFILT", + "BPZ_FLAGFILT", +] + + +def params_default(): + """Params Default. + + Return default parameter values and additional information + about type and command line options. + + Returns + ------- + tuple + parameter dict, short_options dict, types dict, help_strings dict + + """ + params = { + "input": "unions_shapepipe_comprehensive_struc_2024_v1.5.c.hdf5", + "output": "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.5.c.hdf5", + "fits_dir": "UNIONS_DR6", + "checkpoint": "fill_photoz_bands_checkpoint.json", + "n_check_rows": 10, + "check_tol_arcsec": 0.5, + "verbose": False, + } + + short_options = { + "input": "-i", + "output": "-o", + "fits_dir": "-d", + "checkpoint": "-c", + } + + types = { + "n_check_rows": "int", + "check_tol_arcsec": "float", + } + + help_strings = { + "input": "input HDF5 catalogue (no PhotoPipe fields), default={}", + "output": "output HDF5 catalogue (PhotoPipe fields added and filled), default={}", + "fits_dir": "directory with PhotoPipe FITS tiles, default={}", + "checkpoint": "checkpoint JSON file for resume support, default={}", + "n_check_rows": ( + "number of rows per tile whose RA/Dec are compared between HDF5 and" + " FITS to verify row order, 0 to disable, default={}" + ), + "check_tol_arcsec": ( + "maximum angular separation [arcsec] for a row-order check to pass," + " default={}" + ), + } + + return params, short_options, types, help_strings + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def detect_dataset_name(hf): + """Return the first dataset name in an HDF5 file. + + Tries common names first: dat, dat_comb, data. + + Parameters + ---------- + hf : h5py.File + + Returns + ------- + str + Dataset name. + + Raises + ------ + KeyError + If no dataset is found. + + """ + for name in ("dat", "dat_comb", "data"): + if name in hf: + return name + # Fall back to first key + keys = list(hf.keys()) + if not keys: + raise KeyError("HDF5 file contains no datasets.") + return keys[0] + + +def strip_dtype(dtype): + """Strip h5py metadata from a structured numpy dtype descriptor. + + h5py sometimes adds encoding metadata to dtype descriptions, e.g. + ``('= n_rows: + return np.arange(n_rows, dtype=np.int64) + + n_edge = min(5, n_sample // 3) + head = np.arange(n_edge, dtype=np.int64) + tail = np.arange(n_rows - n_edge, n_rows, dtype=np.int64) + + n_mid = n_sample - 2 * n_edge + if n_mid > 0: + mid = np.linspace(n_edge, n_rows - n_edge - 1, n_mid).astype(np.int64) + else: + mid = np.empty(0, dtype=np.int64) + + return np.unique(np.concatenate([head, mid, tail])) + + +def angular_separation_arcsec(ra_1, dec_1, ra_2, dec_2): + """Angular Separation Arcsec. + + Great-circle separation between two sets of sky coordinates, computed + with the haversine formula (numerically stable at small separations and + correct across the RA=0 wrap). + + Parameters + ---------- + ra_1 : numpy.ndarray + Right ascensions of the first set [deg]. + dec_1 : numpy.ndarray + Declinations of the first set [deg]. + ra_2 : numpy.ndarray + Right ascensions of the second set [deg]. + dec_2 : numpy.ndarray + Declinations of the second set [deg]. + + Returns + ------- + numpy.ndarray + Angular separations [arcsec]. + + """ + ra_1 = np.radians(np.asarray(ra_1, dtype=np.float64)) + dec_1 = np.radians(np.asarray(dec_1, dtype=np.float64)) + ra_2 = np.radians(np.asarray(ra_2, dtype=np.float64)) + dec_2 = np.radians(np.asarray(dec_2, dtype=np.float64)) + + d_ra = ra_2 - ra_1 + d_dec = dec_2 - dec_1 + hav = np.sin(d_dec / 2) ** 2 + np.cos(dec_1) * np.cos(dec_2) * np.sin(d_ra / 2) ** 2 + sep_rad = 2 * np.arcsin(np.sqrt(np.clip(hav, 0, 1))) + + return np.degrees(sep_rad) * 3600 + + +def check_tile_row_order( + dset, + sorted_idx, + fits_data, + fits_radec_cols, + n_sample, + tol_arcsec, +): + """Check Tile Row Order. + + Spot-check that the FITS rows of a tile line up with the HDF5 rows they + are about to be written into. ``write_tile_to_hdf5`` assigns FITS row + ``k`` to HDF5 row ``sorted_idx[k]``; this compares the sky positions of + a sample of those pairs. Only the sampled HDF5 rows are read. + + Parameters + ---------- + dset : h5py.Dataset + Compound HDF5 dataset. + sorted_idx : numpy.ndarray + Sorted HDF5 row indices of this tile. + fits_data : numpy.recarray + Data from the PhotoPipe FITS HDU, same length as ``sorted_idx``. + fits_radec_cols : tuple + (RA, Dec) column names in ``fits_data``. + n_sample : int + Number of rows to compare. + tol_arcsec : float + Maximum tolerated angular separation [arcsec]. + + Returns + ------- + tuple + (ok, n_checked, max_sep_arcsec). ``n_checked`` counts the sampled + rows with a valid sky position on both sides; only those are + compared. ``ok`` is ``False`` if any compared pair is further apart + than ``tol_arcsec``, or if the sample holds no comparable row at all + (``n_checked == 0``, ``max_sep_arcsec`` is ``numpy.nan``): a tile + whose order cannot be verified is not written. + + """ + positions = sample_row_positions(len(sorted_idx), n_sample) + if len(positions) == 0: + return False, 0, np.nan + + ra_col_fits, dec_col_fits = fits_radec_cols + + # Fancy-index a handful of rows only; positions is sorted and unique, and + # sorted_idx is sorted, so the h5py selection is strictly increasing. + rows_hdf5 = dset[sorted_idx[positions]] + + ra_hdf5 = np.asarray(rows_hdf5[RA_COL_HDF5], dtype=np.float64) + dec_hdf5 = np.asarray(rows_hdf5[DEC_COL_HDF5], dtype=np.float64) + ra_fits = np.asarray(fits_data[ra_col_fits][positions], dtype=np.float64) + dec_fits = np.asarray(fits_data[dec_col_fits][positions], dtype=np.float64) + + # A row carries no information about the pairing if either side has no + # valid position (NaN, or a sentinel such as -199 outside the sky). + valid = ( + np.isfinite(ra_hdf5) + & np.isfinite(dec_hdf5) + & np.isfinite(ra_fits) + & np.isfinite(dec_fits) + & (np.abs(dec_hdf5) <= 90) + & (np.abs(dec_fits) <= 90) + ) + n_checked = int(valid.sum()) + if n_checked == 0: + return False, 0, np.nan + + sep = angular_separation_arcsec( + ra_hdf5[valid], dec_hdf5[valid], ra_fits[valid], dec_fits[valid] + ) + max_sep = float(np.max(sep)) + + return bool(max_sep <= tol_arcsec), n_checked, max_sep + + +def write_tile_to_hdf5(dset, hdf5_indices, fits_data, valid_keys): + """Write valid_keys from fits_data into dset at hdf5_indices. + + Reads the HDF5 range in one chunk, fills fields in memory, writes + back. Handles both contiguous and non-contiguous index ranges. + + Assumes row ``k`` of ``fits_data`` corresponds to HDF5 row + ``numpy.sort(hdf5_indices)[k]``; ``check_tile_row_order`` spot-checks + this before the write. + + Parameters + ---------- + dset : h5py.Dataset + Compound HDF5 dataset opened in r+ mode. + hdf5_indices : numpy.ndarray + Row indices in dset corresponding to this tile (will be sorted). + fits_data : numpy.recarray + Data from the PhotoPipe FITS HDU. + valid_keys : list of str + Field names to copy from fits_data into dset. + + """ + sorted_idx = np.sort(hdf5_indices) + idx_min = int(sorted_idx[0]) + idx_max = int(sorted_idx[-1]) + n_range = idx_max - idx_min + 1 + + if n_range == len(sorted_idx): + # Contiguous block: single read-modify-write + chunk = dset[idx_min : idx_max + 1] + for key in valid_keys: + chunk[key] = fits_data[key] + dset[idx_min : idx_max + 1] = chunk + else: + # Non-contiguous: split into contiguous sub-blocks + gaps = np.where(np.diff(sorted_idx) > 1)[0] + 1 + blocks = np.split(sorted_idx, gaps) + fits_offset = 0 + for block in blocks: + b_min, b_max = int(block[0]), int(block[-1]) + n_block = b_max - b_min + 1 + chunk = dset[b_min : b_max + 1] + for key in valid_keys: + chunk[key] = fits_data[key][fits_offset : fits_offset + n_block] + dset[b_min : b_max + 1] = chunk + fits_offset += n_block + + +# --------------------------------------------------------------------------- +# Phase 1: create output file +# --------------------------------------------------------------------------- + + +def create_output_file(input_path, output_path, dataset_name, verbose=False): + """Create output HDF5 by copying input and appending empty PhotoPipe fields. + + Parameters + ---------- + input_path : str + output_path : str + dataset_name : str + Dataset name in the input file (used for output too). + verbose : bool + + """ + print(f"Phase 1: creating output file '{output_path}'") + t0 = timer() + + with h5py.File(input_path, "r") as hf_in: + dset_in = hf_in[dataset_name] + n_total = dset_in.shape[0] + dtype_out = build_output_dtype(dset_in.dtype, REQUESTED_KEYS) + new_keys = [k for k in REQUESTED_KEYS if k not in set(dset_in.dtype.names)] + + print(f" Input rows : {n_total:,}") + print(f" Input fields : {len(dset_in.dtype.names)}") + print(f" New fields : {new_keys}") + size_mb = n_total * dtype_out.itemsize / 1_048_576 + print(f" Output size : ~{size_mb:,.0f} MB") + + with h5py.File(output_path, "w") as hf_out: + dset_out = hf_out.create_dataset( + dataset_name, + shape=(n_total,), + dtype=dtype_out, + ) + + # Copy input fields chunk by chunk + input_fields = dset_in.dtype.names + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Copying" + ) as pbar: + for start in range(0, n_total, COPY_CHUNK): + end = min(start + COPY_CHUNK, n_total) + chunk_in = dset_in[start:end] + chunk_out = np.empty(end - start, dtype=dtype_out) + for field in input_fields: + chunk_out[field] = chunk_in[field] + for key in new_keys: + chunk_out[key] = EMPTY_VALUE + dset_out[start:end] = chunk_out + pbar.update(end - start) + + # Copy all other top-level datasets/groups unchanged + for key in hf_in.keys(): + if key != dataset_name: + hf_in.copy(key, hf_out) + if verbose: + print(f" Copied group/dataset '{key}' unchanged.") + + elapsed = timer() - t0 + print(f" Done in {elapsed:.1f}s\n") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(argv=None): + """Main. + + Main program. + + """ + params, short_options, types, help_strings = params_default() + + options = cs_args.parse_options(params, short_options, types, help_strings) + params.update(options) + + logging.log_command(argv) + + verbose = params["verbose"] + + if not os.path.exists(params["input"]): + print(f"ERROR: input file not found: {params['input']}", file=sys.stderr) + return 1 + if not os.path.isdir(params["fits_dir"]): + print(f"ERROR: FITS directory not found: {params['fits_dir']}", file=sys.stderr) + return 1 + + # ------------------------------------------------------------------ + # Load checkpoint + # ------------------------------------------------------------------ + if os.path.exists(params["checkpoint"]): + with open(params["checkpoint"]) as f: + checkpoint = json.load(f) + done_tiles = set(checkpoint.get("done_tiles", [])) + print(f"Resuming: {len(done_tiles)} tiles already completed.") + else: + done_tiles = set() + checkpoint = {} + + # ------------------------------------------------------------------ + # Detect input dataset name (cache in checkpoint) + # ------------------------------------------------------------------ + if "dataset_name" in checkpoint: + dataset_name = checkpoint["dataset_name"] + else: + with h5py.File(params["input"], "r") as hf: + dataset_name = detect_dataset_name(hf) + checkpoint["dataset_name"] = dataset_name + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + if verbose: + print(f"Input dataset: '{dataset_name}'") + + # ------------------------------------------------------------------ + # Phase 1: create output file if needed + # ------------------------------------------------------------------ + if not checkpoint.get("output_created", False): + if os.path.exists(params["output"]): + print( + f"WARNING: output file '{params['output']}' exists but checkpoint " + "does not mark it as complete. Overwriting." + ) + create_output_file( + params["input"], params["output"], dataset_name, verbose=verbose + ) + checkpoint["output_created"] = True + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + else: + if verbose: + print("Phase 1 already done (output file exists in checkpoint).") + + # ------------------------------------------------------------------ + # Phase 2: fill tiles + # ------------------------------------------------------------------ + t0 = timer() + with h5py.File(params["output"], "r+") as hf: + dset = hf[dataset_name] + n_total = dset.shape[0] + print(f"Phase 2: filling tiles in '{params['output']}'") + print(f" {n_total:,} rows, dataset '{dataset_name}'") + + hdf5_fields = set(dset.dtype.names) + + # Check all requested keys are present + for key in REQUESTED_KEYS: + if key not in hdf5_fields: + warnings.warn( + f"Key '{key}' missing from output dataset — was Phase 1 complete?" + ) + + # Build TILE_ID → output row indices (always scanned; too large for checkpoint) + print(" Building tile→index map (scans all rows)...") + tile_index_map_lists = defaultdict(list) + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Scanning TILE_ID" + ) as pbar: + for start in range(0, n_total, SCAN_CHUNK): + end = min(start + SCAN_CHUNK, n_total) + tile_chunk = dset[start:end]["TILE_ID"] + for local_i, tid in enumerate(tile_chunk): + tile_index_map_lists[tid].append(start + local_i) + pbar.update(end - start) + + tile_index_map = { + tid: np.array(idxs, dtype=np.int64) + for tid, idxs in tile_index_map_lists.items() + } + print(f" Map built: {len(tile_index_map)} unique tiles.") + + unique_tiles = sorted(tile_index_map.keys()) + n_tiles = len(unique_tiles) + + valid_keys = None # determined from first available FITS tile + fits_radec_cols = None # idem, (RA, Dec) column names in the FITS tiles + + # The row-order check needs sky positions on both sides. + check_rows = params["n_check_rows"] + if check_rows > 0 and not {RA_COL_HDF5, DEC_COL_HDF5} <= hdf5_fields: + warnings.warn( + f"Columns '{RA_COL_HDF5}'/'{DEC_COL_HDF5}' absent from the HDF5" + " dataset — row-order check disabled." + ) + check_rows = 0 + n_skipped_missing = 0 + n_skipped_done = 0 + n_skipped_size = 0 + n_skipped_order = 0 + n_errors = 0 + n_consec_fails = 0 + n_processed = 0 + + print(f"\n Processing {n_tiles} tiles ({len(done_tiles)} already done)...\n") + pbar = tqdm.tqdm(unique_tiles, total=n_tiles, unit="tile") + + for tile_id in pbar: + if n_consec_fails >= MAX_CONSEC_FAILS: + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + print( + f"\nERROR: {n_consec_fails} tiles failed in a row, " + "likely a systematic problem; aborting.", + file=sys.stderr, + ) + sys.exit(1) + + tile_str = tile_id.decode() if isinstance(tile_id, bytes) else tile_id + + if tile_str in done_tiles: + n_skipped_done += 1 + continue + + fits_path = tile_id_to_fits_path(tile_id, params["fits_dir"]) + if not os.path.exists(fits_path): + n_skipped_missing += 1 + done_tiles.add(tile_str) + pbar.set_postfix({"done": n_processed, "missing": n_skipped_missing}) + continue + + try: + with fits.open(fits_path, memmap=True) as hdu_list: + fits_data = hdu_list[FITS_HDU].data + + # Validate keys on first successfully opened tile + if valid_keys is None: + valid_keys, missing_keys = check_fits_keys( + fits_data.dtype.names, REQUESTED_KEYS + ) + valid_keys = [k for k in valid_keys if k in hdf5_fields] + if missing_keys: + warnings.warn( + f"Keys absent from PhotoPipe FITS (skipped): {missing_keys}" + ) + tqdm.tqdm.write(f"\n Keys to fill: {valid_keys}\n") + + fits_radec_cols = find_fits_radec_columns(fits_data.dtype.names) + if check_rows > 0 and fits_radec_cols[0] is None: + warnings.warn( + "No known RA/Dec column pair in the PhotoPipe" + f" FITS (tried {FITS_RADEC_CANDIDATES}) —" + " row-order check disabled." + ) + check_rows = 0 + elif check_rows > 0: + tqdm.tqdm.write( + " Row-order check: comparing" + f" {check_rows} rows/tile," + f" {RA_COL_HDF5}/{DEC_COL_HDF5} (HDF5) vs" + f" {fits_radec_cols[0]}/{fits_radec_cols[1]}" + f" (FITS), tolerance" + f" {params['check_tol_arcsec']} arcsec\n" + ) + + hdf5_indices = tile_index_map[tile_id] + n_hdf5 = len(hdf5_indices) + n_fits = len(fits_data) + + if n_hdf5 != n_fits: + warnings.warn( + f"Tile {tile_str}: HDF5 has {n_hdf5} rows, FITS has " + f"{n_fits} — size mismatch, skipping." + ) + n_skipped_size += 1 + n_consec_fails += 1 + pbar.set_postfix( + {"done": n_processed, "size_err": n_skipped_size} + ) + continue + + # write_tile_to_hdf5 pairs FITS row k with HDF5 row + # sorted_idx[k]; spot-check that pairing before writing. + sorted_idx = np.sort(hdf5_indices) + if check_rows > 0: + ok, n_checked, max_sep = check_tile_row_order( + dset, + sorted_idx, + fits_data, + fits_radec_cols, + check_rows, + params["check_tol_arcsec"], + ) + if not ok: + if n_checked == 0: + warnings.warn( + f"Tile {tile_str}: no sampled row has a" + " valid RA/Dec on both sides —" + " row order unverifiable, skipping." + ) + else: + warnings.warn( + f"Tile {tile_str}: RA/Dec disagree for" + f" {n_checked} checked rows (max" + f" separation {max_sep:.3g} arcsec >" + f" {params['check_tol_arcsec']} arcsec)" + " — row order mismatch, skipping." + ) + n_skipped_order += 1 + n_consec_fails += 1 + pbar.set_postfix( + { + "done": n_processed, + "order_err": n_skipped_order, + } + ) + continue + + write_tile_to_hdf5(dset, sorted_idx, fits_data, valid_keys) + + except Exception as e: + warnings.warn(f"Tile {tile_str}: error ({e}), skipping.") + n_errors += 1 + n_consec_fails += 1 + continue + + n_processed += 1 + n_consec_fails = 0 + done_tiles.add(tile_str) + + if n_processed % 50 == 0: + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + pbar.set_postfix({"done": n_processed, "missing": n_skipped_missing}) + + # Final checkpoint flush + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + elapsed = timer() - t0 + print(f"\nDone in {elapsed:.1f}s") + print(f" Tiles processed : {n_processed}") + print(f" Tiles skipped (done) : {n_skipped_done}") + print(f" FITS files missing : {n_skipped_missing}") + print(f" Size mismatches : {n_skipped_size}") + print(f" Row-order mismatches : {n_skipped_order}") + print(f" Tiles failed (error) : {n_errors}") + if n_processed == 0 and (n_errors > 0 or n_skipped_size > 0 or n_skipped_order > 0): + print( + "WARNING: no tiles were filled; all available tiles failed.", + file=sys.stderr, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/src/sp_validation/tests/test_fill_photoz_bands.py b/src/sp_validation/tests/test_fill_photoz_bands.py new file mode 100644 index 00000000..ae45a637 --- /dev/null +++ b/src/sp_validation/tests/test_fill_photoz_bands.py @@ -0,0 +1,304 @@ +"""``scripts/fill_photoz_bands.py`` must not trust FITS/HDF5 row order blindly. + +The script writes PhotoPipe columns into the ShapePipe comprehensive HDF5 +catalogue tile by tile, pairing FITS row ``k`` with the ``k``-th HDF5 row of +that tile (in sorted-index order). Nothing in the file formats guarantees +that pairing -- only the row *count* used to be checked -- so a tile whose +PhotoPipe catalogue happens to be ordered differently would be filled with +silently mismatched photo-z. + +These tests build a tiny synthetic pair of catalogues (three tiles, +deliberately interleaved so the non-contiguous write path is exercised) and +run the script end to end: + +* the two tiles whose FITS rows line up are filled; +* the tile whose FITS rows are reversed is *skipped*, counted under + ``Row-order mismatches``, and left at ``EMPTY_VALUE``, so a resumed run + retries it rather than treating it as done. +""" + +import importlib.util +import json +import subprocess +import sys +import warnings +from pathlib import Path + +import numpy as np +import pytest + +h5py = pytest.importorskip("h5py") +fits = pytest.importorskip("astropy.io.fits") +pytest.importorskip("cs_util") +pytest.importorskip("tqdm") + + +def _repo_root() -> Path: + """Locate the repo root by walking up to the ``pyproject.toml`` marker.""" + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +_SCRIPT = _repo_root() / "scripts" / "fill_photoz_bands.py" + +# Tile layout: three tiles, rows interleaved round-robin over the catalogue so +# each tile's HDF5 indices are non-contiguous. +TILES = ("100.100", "200.200", "300.300") +N_PER_TILE = 20 +N_ROWS = len(TILES) * N_PER_TILE +FILL_KEYS = ("Z_B", "MAG_GAAP_r") + + +def _module(): + """Import ``scripts/fill_photoz_bands.py`` (lives outside the package).""" + spec = importlib.util.spec_from_file_location("fill_photoz_bands", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _synthetic_catalogues(tmp_path, reversed_tile): + """Write a synthetic HDF5 catalogue and matching PhotoPipe FITS tiles. + + Parameters + ---------- + tmp_path : pathlib.Path + Directory to write into. + reversed_tile : str + Tile whose FITS rows are written in reverse order (row-order breakage). + + Returns + ------- + tuple + (input HDF5 path, FITS directory, truth dict tile -> (Z_B, MAG_GAAP_r)) + + """ + row = np.arange(N_ROWS) + tile_of_row = np.array([TILES[i % len(TILES)] for i in row]) + + dtype = np.dtype([("TILE_ID", "S7"), ("RA", " 0) + # Not all at the edges: something is sampled from the bulk. + assert np.any((positions > 10) & (positions < 989)) + + +@pytest.mark.fast +def test_find_fits_radec_columns(): + """The DR6 PhotoPipe names are recognised; unknown tables yield None.""" + module = _module() + + assert module.find_fits_radec_columns( + ["SeqNr", "ALPHA_J2000", "DELTA_J2000", "Z_B"] + ) == ("ALPHA_J2000", "DELTA_J2000") + assert module.find_fits_radec_columns(["RA", "DEC"]) == ("RA", "DEC") + assert module.find_fits_radec_columns(["X", "Y"]) == (None, None) + + +@pytest.mark.fast +def test_angular_separation_arcsec(): + """Separation is correct at the pole-free small-angle limit and wraps RA.""" + module = _module() + + sep = module.angular_separation_arcsec([0.0], [0.0], [1.0 / 3600], [0.0]) + np.testing.assert_allclose(sep, [1.0], rtol=1e-6) + + # Across the RA=0 wrap: 359.999 deg vs 0.001 deg is 0.002 deg, not 360. + sep = module.angular_separation_arcsec([359.999], [0.0], [0.001], [0.0]) + np.testing.assert_allclose(sep, [0.002 * 3600], rtol=1e-6) + + +@pytest.mark.fast +def test_check_tile_row_order_ignores_invalid_positions(tmp_path): + """Rows without a valid position on both sides are not compared. + + A NaN or sentinel coordinate carries no information about the pairing, + so it must neither fail the check (NaN <= tol is False) nor pass it; a + sample with no comparable row at all is an unverifiable tile and fails. + """ + module = _module() + + n = 8 + ra = 10.0 + 0.01 * np.arange(n) + dec = 20.0 + 0.01 * np.arange(n) + dtype = np.dtype([("RA", ">f8"), ("Dec", ">f8")]) + data = np.empty(n, dtype=dtype) + data["RA"], data["Dec"] = ra, dec + with h5py.File(tmp_path / "cat.hdf5", "w") as hf: + dset = hf.create_dataset("dat", data=data) + sorted_idx = np.arange(n) + cols = ("ALPHA_J2000", "DELTA_J2000") + + def fits_like(ra_f, dec_f): + rec = np.empty(n, dtype=[(cols[0], " 0.5 + + # Nothing comparable at all: fail, no RuntimeWarning from an all-NaN max. + with warnings.catch_warnings(): + warnings.simplefilter("error") + ok, n_checked, max_sep = module.check_tile_row_order( + dset, sorted_idx, fits_like(np.full(n, np.nan), dec), cols, n, 0.5 + ) + assert not ok and n_checked == 0 and np.isnan(max_sep) + + # A single-row tile is a valid (one-element) fancy index. + ok, n_checked, _ = module.check_tile_row_order( + dset, sorted_idx[3:4], fits_like(ra, dec)[3:4], cols, 10, 0.5 + ) + assert ok and n_checked == 1 From 556e56aa3d835463827383322bf23d3747efa076 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Sat, 5 Sep 2026 16:17:16 +0200 Subject: [PATCH 06/83] cat_config: drop duplicate SP_v1.4.6 / SP_v1.3.6 keys, repair the survivor, guard against recurrence (#320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cat_config: drop duplicate SP_v1.4.6 / SP_v1.3.6 blocks, repair the survivor `cosmo_val/cat_config.yaml` carried two top-level `SP_v1.4.6:` keys and two `SP_v1.3.6:` keys. PyYAML keeps the last, so the second block of each pair was what every consumer saw — and both were stale. They arrived in merge c22f07568, which resolved a conflict by keeping both sides (and renamed the dead `SSP_v1.4.6_msel` key to `SP_v1.4.6` in the process). Delete the shadowing blocks and repair the surviving `SP_v1.4.6` to the post-aa774b65d convention: * `cov_th` stays at A = 2894.03 deg^2 / n_e = 5.0935 (the nside-4096 footprint mask every other live entry uses), not the 2405 / 6.128 pair the shadowing block held. * `shear.redshift_distr` -> `shear.redshift_path`: the code only ever reads `redshift_path`, so the v1.4.6 n(z) was under a key nothing reads while the winning block pointed at the v1.0-era `dndz_SP_A.txt`. * `shear.mask` (read nowhere) -> entry-level `mask:` pointing at `mask_map_footprint_nside_4096.fits`, matching SP_v1.4.5 / SP_v1.4.6.3 and the mask `A` was measured from. * `shear.R: 1.0` — the delivered e1/e2 columns already have the response applied; the shadowing block's `R: 0.92` double-counted it and inflated xi_pm by 1/R^2 = 18% against a covariance that never sees R. `SP_v1.3.6`'s surviving block already carries the nside-4096 footprint mask; its cov_th still holds the old 2405 / 6.128 pair and needs regenerating separately. * tests: reject duplicate keys in cat_config.yaml `yaml.safe_load` silently keeps the last of a repeated key, so a merge that resolves a conflict by keeping both sides leaves the shadowed block invisible to every consumer *and* to every test. That is how two `SP_v1.4.6` and two `SP_v1.3.6` entries survived on develop. Add a `yaml.SafeLoader` subclass whose `construct_mapping` raises on a repeated key, and assert `cosmo_val/cat_config.yaml` parses clean with it. Fails on the pre-fix config with `duplicate key 'SP_v1.3.6' at line 522 (first seen at line 148)`. * cat_config: recompute SP_v1.3.6 cov_th (A, n_e, sigma_e) for the nside-4096 footprint aa774b65d repointed SP_v1.3.6's mask to the 2894 deg² footprint but left its cov_th holding v1.4.6's old numbers. Recomputed from v1.3.6's own catalogue with sp_validation.survey; the same code path reproduces SP_v1.4.6.3's committed A / n_e / sigma_e bit-for-bit. n_psf left as it was. --- cosmo_val/cat_config.yaml | 96 +------------------ .../test_cat_config_no_duplicate_keys.py | 72 ++++++++++++++ 2 files changed, 77 insertions(+), 91 deletions(-) create mode 100644 src/sp_validation/tests/test_cat_config_no_duplicate_keys.py diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index cae6411b..49cdfcc9 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -153,10 +153,10 @@ SP_v1.3.6: ls: dashed marker: h cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 + A: 2894.0303815287743 + n_e: 4.002931781928292 n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 + sigma_e: 0.3785853255980945 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: PSF_flag: HSM_FLAG_PSF @@ -445,6 +445,7 @@ SP_v1.4.6: n_e: 5.09348802763124 n_psf: 0.752316232272063 sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: PSF_flag: HSM_FLAG_PSF PSF_size: HSM_T_PSF @@ -461,8 +462,7 @@ SP_v1.4.6: shear: R: 1.0 path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - redshift_distr: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt - mask: v1.4.6/footprint_binned_8192.fits + redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -519,92 +519,6 @@ SP_v1.4.6.3: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.3.6: - subdir: /n17data/UNIONS/WL/v1.3.x - pipeline: SP - colour: green - getdist_colour: 0.0, 0.5, 1.0 - ls: dashed - marker: d - cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 - n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits - psf: - PSF_flag: HSM_FLAG_PSF - PSF_size: HSM_T_PSF - star_flag: HSM_FLAG_STAR - star_size: HSM_T_STAR - hdu: 1 - path: unions_shapepipe_psf_2022_v1.3.a.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: HSM_G1_PSF - e1_star_col: HSM_G1_STAR - e2_PSF_col: HSM_G2_PSF - e2_star_col: HSM_G2_STAR - shear: - R: 1.0 - path: v1.3.6/unions_shapepipe_cut_struc_2022_v1.3.6.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - w_col: w_des - e1_col: e1 - e1_col_corrected: e1_leak_corrected - e1_PSF_col: e1_PSF - e2_col: e2 - e2_col_corrected: e2_leak_corrected - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3.fits -SP_v1.4.6: - subdir: /n17data/UNIONS/WL/v1.4.x - pipeline: SP - colour: green - getdist_colour: 0.0, 0.5, 1.0 - ls: dashed - marker: d - cov_th: - A: 2405.3892055695346 - n_e: 6.128201234871523 - n_psf: 0.752316232272063 - sigma_e: 0.379587601488189 - mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits - psf: - PSF_flag: HSM_FLAG_PSF - PSF_size: HSM_T_PSF - star_flag: HSM_FLAG_STAR - star_size: HSM_T_STAR - hdu: 1 - path: unions_shapepipe_psf_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: HSM_G1_PSF - e1_star_col: HSM_G1_STAR - e2_PSF_col: HSM_G2_PSF - e2_star_col: HSM_G2_STAR - shear: - R: 0.92 - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - w_col: w_des - e1_col: e1 - e1_col_corrected: e1_leak_corrected - e1_PSF_col: e1_PSF - e2_col: e2 - e2_col_corrected: e2_leak_corrected - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.6.3_B: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP diff --git a/src/sp_validation/tests/test_cat_config_no_duplicate_keys.py b/src/sp_validation/tests/test_cat_config_no_duplicate_keys.py new file mode 100644 index 00000000..1ab0df2e --- /dev/null +++ b/src/sp_validation/tests/test_cat_config_no_duplicate_keys.py @@ -0,0 +1,72 @@ +"""Guard: ``cosmo_val/cat_config.yaml`` has no repeated mapping keys. + +PyYAML accepts duplicate keys silently, keeping the last occurrence. A merge +that resolves a conflict by keeping both sides therefore produces a config +where the *shadowed* block is invisible to every consumer and to every test +that loads it with ``yaml.safe_load`` -- which is how two ``SP_v1.4.6`` and +two ``SP_v1.3.6`` entries lived on ``develop`` with the stale ones winning. + +This loads the catalogue config with a loader that raises instead. + +:Author: cdaley + +""" + +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.fast + + +class DuplicateKeyError(ValueError): + """Raised when a YAML mapping repeats a key.""" + + +class UniqueKeySafeLoader(yaml.SafeLoader): + """``yaml.SafeLoader`` that rejects repeated keys instead of overwriting.""" + + def construct_mapping(self, node, deep=False): + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if key in mapping: + raise DuplicateKeyError( + f"duplicate key {key!r} at line {key_node.start_mark.line + 1} " + f"(first seen at line {mapping[key] + 1})" + ) + mapping[key] = key_node.start_mark.line + return super().construct_mapping(node, deep=deep) + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def load_unique(path: Path): + """Parse ``path`` as YAML, raising ``DuplicateKeyError`` on repeated keys.""" + with Path(path).open() as handle: + return yaml.load(handle, Loader=UniqueKeySafeLoader) + + +def test_loader_rejects_duplicate_keys(tmp_path): + """The loader itself catches a repeated key (and ``safe_load`` does not).""" + config = tmp_path / "dup.yaml" + config.write_text("a:\n x: 1\nb:\n y: 2\na:\n x: 3\n") + + assert yaml.safe_load(config.read_text()) == {"a": {"x": 3}, "b": {"y": 2}} + with pytest.raises(DuplicateKeyError, match="duplicate key 'a'"): + load_unique(config) + + +def test_cat_config_has_no_duplicate_keys(): + """``cosmo_val/cat_config.yaml`` parses with no key shadowing another.""" + config_path = _repo_root() / "cosmo_val" / "cat_config.yaml" + assert config_path.exists(), f"missing config: {config_path}" + + config = load_unique(config_path) + assert config, "cat_config.yaml parsed empty" From 7dd19847133a5d81244fa6ad1e37b3b97743ae1b Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Tue, 8 Sep 2026 16:11:39 +0200 Subject: [PATCH 07/83] Improvements to snakemake containerization (#309) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixes from the snakemake inference run report (#300) Rules: drop the /automnt prefix from the hardcoded paths in xi_highres (twopoint.smk) and covariance_glass_mock (covariance.smk). /automnt/nXXdataN does not exist on the node that owns that disk, so a job landing there fails immediately, before any log is written. Every canonical path in common.py already uses the plain /nXXdataN form. Docs: workflow/README.md gains a note on the /automnt trap and one on host ~/.local shadowing the container's pinned Snakemake; cosmo_inference/README.md recommends CosmoSIS --mpi over the fragile upstream --smp process pool (cosmosis#170) and cosmosis >= 3.16.1. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MS3u2QuVSK1Q2CVaxcPNxU * workflow: profile-driven apptainer containerization (set A) candide profile now owns the container: software-deployment-method: apptainer + apptainer-args carries the bind mounts (matching the app bash-function binds in the top-level UNIONS CLAUDE.md), replacing the old rationale for leaving containerization to each rule. Rewrites the profile's doc comment to describe the new model and its two documented exceptions (xi_highres MPI, covariance_cosmocov host toolchain). Adds a container_smoke rule (workflow/rules/container_smoke.smk + scripts/container_smoke.py) as a cheap end-to-end check of the profile-driven container path (editable sp_validation import, numpy + OMP_NUM_THREADS, git provenance) via `script:`, wired unconditionally into workflow/Snakefile. Reconciles image_sims/Snakefile's container: None comment: it now documents that the two-image (SIF/SIF_PIPELINE) chain is a per-rule container: choice in image_sims.smk, still wrapped by the profile's apptainer deployment -- not a rule-owned apptainer exec call. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MS3u2QuVSK1Q2CVaxcPNxU * workflow: profile-driven apptainer containerization (set B: image_sims) Strip every rule's explicit apptainer exec wrapper (_EXEC_PREFIX/EXEC/ EXEC_PIPELINE) from image_sims.smk. Each compute rule now carries a plain per-rule container: SIF / container: SIF_PIPELINE directive; Snakemake wraps the shell: command via the profile's software-deployment-method: apptainer + apptainer-args (set A). PYTHONPATH/PSF_DICT/OMP_NUM_THREADS injection and the SLURM_* env strip move from apptainer --env/-u flags to plain shell VAR=value / env -u syntax at the front of each shell: string (_ENV_PREFIX) -- identical effect, no apptainer-specific mechanism, works the same whether or not the command is container-wrapped. im_mbias split into im_mbias_config (run:, host-side git/provenance introspection + yaml write -- Snakemake never containerizes run: regardless of container:, so this must stay a driver-side step) and im_mbias (shell:, container: SIF, runs the actual m-bias compute). This was the one rule whose apptainer call lived inside a run: block's trailing shell() -- splitting it out is what makes container: apply to it at all. binds: dropped from image_sims config/schema -- it's now the profile's apptainer-args (one bind list for the whole workflow), not a per-run-config value. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MS3u2QuVSK1Q2CVaxcPNxU * workflow: profile-driven apptainer containerization (set C: remaining rules + docs) Completes the pivot for the rules outside image_sims: xi_highres and covariance_cosmocov keep their container: None + inline apptainer exec / host-toolchain call (multi-node MPI and a host-compiled binary respectively, both genuinely incompatible with Snakemake's own container wrapping), now documented in their rule docstrings as deliberate exceptions rather than leftovers. No other rule outside image_sims called apptainer directly. While touching these files, retired the stale /pure_eb/ absolute paths left from the old repo layout: added workflow.common.WORKFLOW_SCRIPTS (Path(__file__)-based, correct under both standalone and module-composed runs) for the handful of shell: rules that call a workflow script directly, and reused the existing COSMO_INFERENCE constant elsewhere. Removed the run_cosmo_val rule in twopoint.smk, dead since cosmo_val.smk decomposed it into per-diagnostic rules (its own docstring says so) and still pointing at a stale path plus a nonsensical host .local PYTHONPATH injection. Flagged (not fixed) covariance_process: it calls cosmo_inference/scripts/ cosmocov_process.py, deleted in the #236 cleanup and never restored, so the rule fails on the default covariance target -- pre-existing, unrelated to this pivot. Rewrote workflow/README.md and cosmo_inference/README.md to the new model: snakemake is a thin host-side tool pinned via `uv tool install snakemake snakemake-executor-plugin-slurm`, run directly on the host, never from inside an apptainer shell; the candide profile's software-deployment-method puts each job in the container instead. Added a short pointer from the top-level README's dev-shell instructions to workflow/README.md so the two don't get conflated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MS3u2QuVSK1Q2CVaxcPNxU * workflow: fix script: directive under the profile-driven container container_smoke failed for real (SLURM jobs 839818/839819) with ModuleNotFoundError: snakemake.iocontainers -- the container image had its own snakemake==9.16.3 pip-installed directly (leftover from the old apptainer-shell-then-snakemake-inside pattern this pivot retires), shadowing the host-mounted 9.23.1 orchestrator that script: bind-mounts in and sys.path.extends (appended, not prepended). Removed it and its snakemake-executor-plugin-slurm/-slurm-jobstep/-interface-* family from the image (verified Required-by: none outside the family itself). Separately, apptainer-args never actually isolated host tooling: the image's own /.singularity.d/env/50-bashrc.sh unconditionally sourced the host ~/.bashrc for every apptainer action, not just an interactive `apptainer shell` -- so a host dotfile (asdf init) ran on every exec too, pushing host PATH entries (~/.local/bin) ahead of the image's own /usr/local/bin. A bare `python` in any shell:/script: rule was silently running the host's interpreter, invisibly, surviving --cleanenv. Gated the bashrc sourcing on APPTAINER_COMMAND=shell (set by apptainer itself before these scripts run). Fixing both surfaced a third, previously-masked bug: every script: rule (19 files) imports `from snakemake.script import snakemake`, which is IDE-hint-only in this snakemake version -- snakemake.script exposes no such runtime attribute (only the Snakemake class), and the preamble that actually gets pickled in already provides `snakemake` as a plain global before the rest of the file executes. Removed the broken import repo-wide; the object resolves via normal global lookup exactly as before, including inside the functions/branches a few scripts defer it into. Verified end to end: container_smoke now completes for real through SLURM (jobid 839822, python 3.12.12, sp_validation editable install resolved, correct HEAD commit read from inside the job) with no apptainer exec left in any rule. Dry-run coverage for every rule that owns an edited script (masks_only, cv_weights, and friends) shows clean DAGs. The container-image invariants (no in-image snakemake, exec/run must not source host dotfiles) aren't reproducible from this repo -- the sandbox at /n17data/cdaley/containers/containers has no tracked build recipe -- so they're now documented in workflow/README.md for the next rebuild. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MS3u2QuVSK1Q2CVaxcPNxU * workflow: remove last live apptainer-exec shell call + stale sweep-script docs presentation_pte_cosebis was the one rule left with an inline apptainer exec in its shell: block (container: None override); every sibling presentation_* rule already relies on the module-level container default. Drop the override and the raw call so it's wrapped like the rest. Also update the three sweep-script docstrings (run_xi_sweep, run_cosebis_ptes_sweep, run_cl_sweep) whose example invocations still showed the retired apptainer-exec-then-python pattern, to match the plain `python script.py ...` convention already used by their sibling CLI scripts. * profile: add --cleanenv to apptainer-args Per-node smoke tests showed default env passthrough makes container python resolution nondeterministic (host ~/.local shadowing — the #302 mechanism). --cleanenv makes every job's environment container-defined. Verified: container_smoke green through the profile with it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MS3u2QuVSK1Q2CVaxcPNxU * tests: adapt pure-E/B and glass-mock tests to the tomographic API calculate_pure_eb now returns one results dict per tomographic bin pair (#297); the pure-E/B integration test still indexed the flat mode keys. Unwrap the non-tomographic "tomo_bin_all_tomo_bin_all" entry and correct the docstring that still advertised the flat return. glass_mock's map path imports cosmology.compat.camb, which is absent in the image, so the xfail's raises=AttributeError no longer matched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QndBZicN3QvyZG4XDDPmGs * docs: sweep comment bloat across the pivot diff Remove the 'snakemake is injected' comment repeated in 19 script: files (one note in workflow/README.md instead), trim the candide profile header to the operational lessons, and cut re-narrations of the container model in Snakefile/common.py/README. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * image_sims: one image, drop dead SLURM env strip, im_mbias_config as script Collapse sif/sif_pipeline to the single sp_validation image (it ships shapepipe; must be rebuilt from uv.lock — the current 2026-07-04 image predates the lock and its numpy 2.5 breaks numba/ngmix). Remove the env -u SLURM_* prefix: shapepipe#744 gates mpi4py on OMPI/PMI vars, and --cleanenv strips the host env anyway (verified in-container). Convert im_mbias_config from a run: block to script:, drop the redundant os.makedirs, and trim pivot re-narration from comments. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * covariance: restore cosmocov_process.py as a containerized script: rule Deleted in the #236 cleanup with no replacement; restored from history to workflow/scripts/ (the rule is its only caller) and converted the rule from shell: to script:. Fixes on the way: bare exit() on a non-PD matrix returned 0 (Snakemake saw success) — now sys.exit(1); eigvalsh for the symmetric matrix; Agg backend; plot dpi 2000 -> 300. Verified round-trip on synthetic input in the container. Drop the NOTE and stale comments. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * tests: container_smoke becomes a real pytest, out of the main workflow The rule asserted nothing and put a non-scientific artifact in every paper's results/. Now: tests/data/container_smoke/{Snakefile,script} driven by test_container_smoke.py (@slow, skipped off-cluster), which submits one tiny SLURM job through the committed candide profile and asserts APPTAINER_CONTAINER is set (the job really ran in the image), the editable install resolved, seed-42 eigh values match, and git works inside the container. OMP_NUM_THREADS is recorded, not asserted — unset is the profile's designed state. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * workflow: locate shell-invoked scripts via workflow.source_path Replace the hand-rolled WORKFLOW_SCRIPTS constant with Snakemake's first-class mechanism, which the docs specifically prescribe because manual path construction breaks under module composition. The script goes in input: (not params:, which would cause spurious reruns), so it also becomes an honest dependency. Fixes a live bug on the way: papers/bmodes unblinding_ceremony called 'python workflow/scripts/unblinding_ceremony.py', which from the paper workdir resolves into the paper's own scripts/ dir -- stale since 091bba8 and only detectable at run time. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * workflow: use script: for single-process rules, source_path only for MPI script: resolves relative to the .smk defining it, module composition included, so it defeats the paper-workdir trap without making scripts into input files -- and matches what the other 20+ rules already do. Only xi_highres keeps source_path, where script: is structurally impossible (snakemake would wrap the whole mpiexec line in one container). unblinding_ceremony was already written for script: -- its _config_from_snakemake was dead code because the rule invoked it via shell:, so it silently ran _config_from_cli, which re-derives paths from constants that no longer exist (a cosmo_val dir deleted from the referenced checkout, and a _PROJECT_ROOT off by one since the script moved to papers/bmodes/scripts/). The rule declared 6 inputs and 9 params while passing 2 on the command line. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * papers/bmodes: delete the unblinding ceremony Never used, and silently broken: the rule invoked the script via shell:, so the script's snakemake branch was dead code and its CLI branch re-derived paths from constants that no longer exist. Nothing imported it and no rule consumed its output. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * containers: run the CI-published image from one shared path CI builds ghcr.io/cosmostat/sp_validation on every push from uv.lock; the hand-built SIFs it replaces were stale in ways that only failed at run time (no shapepipe.modules in one, numpy 2.5 breaking numba in the other). Every call site -- the Snakefiles, image_sims, the MPI rule's own apptainer exec, the paper shell drivers, interactive use -- now names one file, refreshed deliberately (see workflow/README.md). Overridable with --config container=. im_mbias_config read the OCI revision by opening the configured sif path; it now reads APPTAINER_CONTAINER, since the image a job actually ran in may be overridden and current.sif is a moving target. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * deps: ship CosmoSIS in the image, drop the vestigial cosmology pin CosmoSIS was an undeclared, user-supplied dependency of the inference step -- which is why #303 was hand-patched in someone's ~/.local. It pip-installs into the image against the base gfortran/GSL/cfitsio in ~2 min, so declare it. MPIFC must be set at build time or the sampler Makefiles silently skip the MPI targets and --mpi fails at load; chains must run under MPI because the upstream --smp pool is still broken at 3.25.2. cosmology 2022.10.9 was vestigial: the cosmology.compat.camb adapter comes from cosmology-compat-camb via glass[examples]. Relocking drops it and nothing else. UV_PYTHON pins uv to the image's own interpreter, since $HOME is bind-mounted and uv would otherwise pick a host CPython carrying none of the stack. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * containers: pull the CI image by tag, park the MPI rule Snakemake pulls docker://ghcr.io/cosmostat/sp_validation:develop into a shared apptainer prefix on first use and never again, so no digest or path is written down. One constant, CONTAINER_URI in workflow/common.py, is the single source of truth; host-side callers that need a concrete file (the paper shell drivers, interactive use) derive it via workflow/scripts/container_path.py. xi_highres is parked as a comment block: it has never been runnable -- its shell is a bare 'python run_2pcf_highres.py' while the script requires --cat-config and --out -- and parking it leaves covariance_cosmocov as the workflow's only container exception. The MPI reasoning is preserved in the block for whoever revives it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * tests: drop the glass map-path xfail, its condition is met The xfail asked for a compatible glass+cosmology pair verified in a fresh image. glass 2026.2 with cosmology-compat-camb is that pair: the map path runs end to end (11 shells, 66 spectra, monotonic kappa accumulation), so the marker now only hides regressions. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * docker: install liblapack-dev, pin MPIFC absolutely cosmosis's bundled MultiNest links -llapack and the base image ships only the runtime liblapack.so.3 with no dev symlink, so the build died at 'cannot find -llapack'. It passed on candide only because that sandbox had liblapack-dev installed at some point. MPIFC takes the absolute path: /opt/ompi/bin is not always on PATH, and a miss silently drops the MPI sampler libraries while the install still reports success. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * docker: point UV_PYTHON at the venv, not the base interpreter uv pip honours UV_PYTHON over VIRTUAL_ENV, so naming the system interpreter sent the editable install of sp_validation there instead of /app/.venv -- the image built fine and then failed its own import smoke test. The venv's own python satisfies the original intent (uv can't wander onto a host CPython from the bind-mounted $HOME) while keeping uv pip pointed at the venv. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017hCncxwiiBZm9ixGgdzCNA * workflow: run the launched checkout's sp_validation by default Snakemake's `script:` directive already executes the checkout's script files, while `import sp_validation` resolved to the image's baked copy -- the two halves of one commit, split. `common.inject_checkout_pythonpath()` prepends the checkout's src/ to APPTAINERENV_PYTHONPATH (preserving any user-set value), so the image supplies the frozen dependency stack and the launched tree supplies sp_validation. This is what the image-sims chain has always done for both repos (`_ENV_PREFIX` in rules/image_sims.smk); the main workflow now matches it. Opt out with `--config checkout_pythonpath=false` to reproduce from the image alone. The flag is parsed tolerantly because `--config k=false` can arrive as the string "false". Also drops the hardcoded candide OpenMPI path from workflow/Snakefile: a machine path does not belong in generic workflow code, and it moves to the candide profile in a following commit. Co-Authored-By: Claude Fable 5 * profiles: add a machine-independent default, make candide the machine layer workflow/profiles/default carries the container model and nothing else, so the workflow runs off candide with `--profile workflow/profiles/default -j N`. Snakemake cannot compose profiles (one --profile, no `inherits:`), so the small machine-independent set -- software-deployment-method, rerun-triggers, latency-wait -- is duplicated verbatim in both files, marked GENERIC and cross-referenced. That is the least-magic arrangement available. apptainer-args and apptainer-prefix stay out of the shared block: every machine has its own disks and image cache. candide's apptainer-args now carries `--env LD_LIBRARY_PATH=/softs/openmpi/...`, previously an os.environ line in workflow/Snakefile. Co-Authored-By: Claude Fable 5 * presentation: containerize the two rules that ran bare python The Moriond talk-figure rules called `python` with no container, so they ran against whatever interpreter the driver happened to have. Let them inherit the module-level `container:` like every other rule. The two ImageMagick `convert` rules keep `container: None`: `convert` is a host tool, absent from the image. Same for covariance_cosmocov, whose docstring says so directly rather than pointing at a list elsewhere. Co-Authored-By: Claude Fable 5 * image_sims: default sif to null and fall back to the workflow's image `image_sims: {sif: ...}` was a required structural key, so every run config repeated the image path — a second place for it to drift from what the rest of the workflow runs. Default it to null and resolve it through the same code path as every other entry point; a run config still overrides it to name its own image or a branch tag. Co-Authored-By: Claude Fable 5 * containers: give every user their own image, driven by spv-container The workflow ran out of one shared image directory on candide (/n17data/cdaley/containers/snakemake-sif) with a hand-maintained current.sif symlink pointing at whatever Snakemake last autopulled. That only worked for one person: refreshing the image or repointing the symlink needed write access to another user's directory, and a refresh moved the ground under everyone at once. Everyone now runs their own image file at one canonical per-user path, ~/.cache/sp_validation/sp_validation.sif (SPV_CONTAINER overrides it), owned by a small CLI: spv-container pull # fetch the tag there, atomically spv-container status # revision label vs. this checkout's HEAD spv-container exec # one-off run inside it, candide binds applied sp_validation.container is stdlib-only on purpose: it runs on the host, outside the container, so it must import without the scientific stack — and it works straight from a checkout (`python3 src/sp_validation/container.py status`) with nothing installed. It also holds CONTAINER_URI, which workflow/common.py loads from this checkout by file path, so the CLI and the workflow can never name different images. `container:` now resolves to that local .sif when it exists and to the registry tag otherwise (Snakemake accepts either, and autopulls the tag into .snakemake/singularity). `--config container=...` still overrides both. The candide profile drops apptainer-prefix accordingly, and common.configure() warns — once, never fatally — when the local image predates the checkout. Also drops workflow/scripts/container_path.py, which existed to locate the shared cache. Co-Authored-By: Claude Fable 5 * docs: tell the container story once, around the per-user image Rewrites the container sections of workflow/README.md, CLAUDE.md, CONTRIBUTING.md and README.md for the per-user model: one image per person at ~/.cache/sp_validation/sp_validation.sif, `spv-container` to fill and inspect it, and how `container:` resolves to it. The shared-prefix machinery is gone -- current.sif bootstrap, the atomic-mv refresh recipe, the group-writable TODO. Trims the commentary while there. The profile pair says "change one, change the other" once instead of shouting it in three places; off-candide gets a paragraph rather than parallel billing, since candide is where everyone runs; and the enumeration of container exceptions is dropped in favour of the docstring on each rule that opts out. Also repoints the two paper Snakefiles, which resolve `container:` themselves, at common.resolve_container -- and replaces the obsolete `apptainer build --sandbox` recipe in README.md and installation.rst with `apptainer pull`. Co-Authored-By: Claude Fable 5 * containers: add an opt-in writable sandbox, and one resolution order The pristine SIF is read-only, which is what you want almost always -- but it lost the one real advantage of the old hand-built sandbox workflow: `pip install` mid-analysis, when you need a package the image does not carry yet and a CI rebuild is too slow a loop to think in. `spv-container sandbox` unpacks the image into a writable directory at ~/.cache/sp_validation/sandbox/, and `spv-container exec --writable` runs against it so installs persist. Opt-in: nothing builds one for you. Resolution order is now one thing, shared by the CLI, the run_*.sh drivers and the workflow's `container:` -- sandbox if it exists, else SIF if it exists, else the registry tag. Snakemake execs a sandbox directory as happily as a .sif, so a package installed into the sandbox is there for workflow jobs too, with no further wiring. `resolve_image()` in sp_validation.container is the single implementation; common.resolve_container defers to it. The build stages into a sibling directory and swaps it in, as `pull` does, for a sharper reason than pull has: a half-written .sif fails loudly, but a half-unpacked sandbox is still a *directory*, so resolution would elect it and every job would silently run a broken tree. Building before removing also means a `--force` rebuild that fails -- a typo in --source, a network blip -- leaves the sandbox you already had intact, instead of deleting a working environment on the way to not replacing it. The cost of a sandbox is that what runs is no longer fully described by a revision label, so the divergence is made visible rather than left silent: `status` names which layer is live and says the revision only describes what the sandbox was built from (falling back to the SIF's label, marked as inferred, when the sandbox carries none), and the workflow prints one line at launch when a sandbox is in play. `spv-container pull && spv-container sandbox --force` resets. Verified on candide (apptainer 1.5.3): unprivileged `build --sandbox` works through user namespaces with no fakeroot and no subuid mapping; `exec --writable` persists writes while plain `exec` gets a read-only filesystem; `inspect --labels` still reports the source image's OCI labels from a sandbox directory; `--fix-perms` at build time is what keeps the tree removable afterwards (without it apptainer leaves directories that defeat `rm -rf`, which would strand `--force`); and a failed `--force` rebuild leaves the existing sandbox and its contents untouched, with no staging directory left behind. Co-Authored-By: Claude Fable 5 * image: build the cosmosis-standard-library fork into the container The cosmo_inference .ini templates pointed COSMOSIS_DIR at two different people's home directories (/home/guerrini/... and a scratch path of Lisa's), so running the inference pipeline meant either being one of them or editing the templates by hand. CosmoSIS itself already ships in the image via the `workflow` extra; only the Standard Library — the tree of module files the pipelines name — was missing. Clone and build it at /opt/cosmosis-standard-library, pinned to Sacha Guerrini's fork at b26fa7ff. That fork is 4 commits ahead of upstream and 373 behind; the four are what the UNIONS pipelines need (tau statistics, sample_S8, two z-dependent linear-alignment modules). Carrying them onto current upstream is future work, noted in cosmo_inference/README.md. The templates now read COSMOSIS_DIR from %(CSL_DIR)s, which the image sets — CosmoSIS reads environment variables into an ini's [DEFAULT] section, which is how the existing %(SCRATCH)s references already work. Off-image, export CSL_DIR and the same templates work unchanged. The build follows CSL's documented procedure for a pip-installed cosmosis (`source cosmosis-configure && make`), but targets `shear/` rather than the top-level `make`: the top level also descends into likelihood/, building the Planck, WMAP and ACT likelihoods, which no UNIONS pipeline uses. Of the modules our templates do name, all are pure Python except two under shear/ — `limber`, which project_2d.py links, and cl_to_xi_nicaea's nicaea_interface.so. Co-Authored-By: Claude Fable 5 * docs: prune duplicated and historical comments The container model, the checkout-PYTHONPATH default, the profile GENERIC mirroring and the CSL_DIR resolution were each explained in three to six places. Give every concept one home -- workflow/README.md for the user-facing story, the docstring of the thing itself for mechanism -- and leave pointers elsewhere. Drop comments narrating what the code used to do; git holds that. Co-Authored-By: Claude Fable 5 * simplify: collapse duplication added by this branch The container model landed the same few lines in several places; fold each into one home. * the four run_*.sh sweep drivers resolved this user's image (and repeated the bind list) inline -- now one sourced papers/bmodes/scripts/container_env.sh, resolving exactly as sp_validation/container.py does (sandbox first, then SPV_CONTAINER/XDG_CACHE_HOME, binds from SPV_APPTAINER_BINDS) * container.py: one _require_apptainer() instead of three copies of the PATH guard, and compare_revision's merge-base calls go through _git * common.py: resolve_container takes the override value, so image_sims.smk no longer wraps IMSIM["sif"] in a synthetic config dict; drop the CONTAINER_URI / local_sif / local_sandbox re-exports, which have no callers * cosmocov_process.py: only Snakemake runs it, so drop the argv entry point and its main() indirection, matching im_mbias_config.py * xip_xim.py: one catalog() builder for the tomographic and non-tomographic paths instead of two near-identical treecorr.Catalog blocks Co-Authored-By: Claude Fable 5 * simplify: give the sweep drivers one shared preamble; fold a duplicated README section The four papers/bmodes sweep drivers each repeated the worktree path, the derived script/source dirs, and a full `apptainer exec ... /usr/local/bin/python` invocation (six sites). container_env.sh now owns all of it and exposes `spv_python` / `sweep_versions`; argv is byte-identical. workflow/README.md explained `--config container=` twice, once for a local .sif and once for a branch tag. One subsection now covers both. Co-Authored-By: Claude Fable 5 * image: actually build CSL — cosmosis-configure exits 0 under set -u without running make Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C856B4eJ3LwXrj9SCiEuMc * image: drop set -u in the CSL layer — the configure exports append to unset paths The test -f artifact guard is what keeps a no-op build loud; -u had become the thing breaking the build instead. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C856B4eJ3LwXrj9SCiEuMc * image: point limber's Makefile at Debian's GSL (GSL_INC/GSL_LIB) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C856B4eJ3LwXrj9SCiEuMc * rebase onto develop: shed tomography-branch remnants The container/workflow work is orthogonal to the tomography branch it was accidentally based on. Restore develop's pure_eb docstring, test_cosmo_val call shape, and glass_mock xfail; keep develop's glass==2025.1 pinned set (cosmology 2022.10.9 is load-bearing there, not vestigial) and relock. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0143zcEsfSWr13AfSroMteEC * docs: make spv-container the install story README leads with the four-line install (clone, symlink onto PATH, pull, exec-check); container.py gets a shebang + exec bit so the symlink is a real CLI with no packaging. installation.rst carries the depth (subcommands, per-user model, sandbox, raw apptainer/docker); CONTRIBUTING and workflow/README point at the same symlink step. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0143zcEsfSWr13AfSroMteEC * CSL runtime coverage: import-test template modules; patch fork for scipy>=1.15; add fast-pt The image build only compiled CSL; pure-Python modules were never loaded until a pipeline ran. Two runtime breaks shipped in a green image: scipy>=1.15 removed scipy.special.lpn (legendre.py, reached by every real-space likelihood via spec_tools -- #316), and project_2d.py imports fastpt, which the image never carried. - test_csl_modules.py imports every .py module the ini templates reference, in-image (skips without CSL_DIR/cosmosis) - Dockerfile applies upstream a8a941d5 (lpn fix) as a patch until the fork absorbs it (#316) - fast-pt>=3.2,<4 joins the workflow extra (4.0 restructured; CSL expects 3.x) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM * ruff autofix (format + safe lint fixes) Pushed by the lint gate. * Repoint CSL at the UNIONS-WL org fork; trim fast-pt annotation Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM * ruff autofix (format + safe lint fixes) Pushed by the lint gate. * lint: noqa E402 on the in-section container-smoke import * Drop CSL scipy workaround after fork merge Pin the container to the current UNIONS-WL fork main commit, remove the now-obsolete lpn patch, and update the inference documentation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XTNCvVQNLXZVTiDR1PZaVG --------- Co-authored-by: Claude Fable 5 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .gitignore | 4 + CLAUDE.md | 20 +- CONTRIBUTING.md | 46 +- Dockerfile | 65 +- README.md | 41 +- cosmo_inference/README.md | 34 +- .../templates/cosmosis_pipeline_A_ia.ini | 4 +- .../templates/cosmosis_pipeline_A_ia_cell.ini | 4 +- .../templates/cosmosis_pipeline_A_psf.ini | 4 +- docs/source/installation.rst | 57 +- papers/bmodes/Snakefile | 5 +- papers/bmodes/rules/presentation.smk | 24 +- papers/bmodes/rules/synthesis.smk | 36 +- papers/bmodes/scripts/container_env.sh | 45 + .../scripts/filter_catalog_ellipticity.py | 1 - .../bmodes/scripts/plot_pure_eb_covariance.py | 2 - papers/bmodes/scripts/run_cl_sweep.py | 2 +- .../bmodes/scripts/run_cosebis_ptes_sweep.py | 2 +- papers/bmodes/scripts/run_cov_sweep.sh | 12 +- .../bmodes/scripts/run_pure_eb_ptes_sweep.sh | 15 +- .../scripts/run_pure_eb_semianalytic.sh | 15 +- papers/bmodes/scripts/run_pure_eb_sweep.sh | 12 +- papers/bmodes/scripts/run_xi_sweep.py | 2 +- papers/bmodes/scripts/unblinding_ceremony.py | 1389 ----------------- papers/cosmo_val/Snakefile | 5 +- pyproject.toml | 23 +- src/sp_validation/container.py | 390 +++++ .../tests/data/container_smoke/Snakefile | 21 + .../data/container_smoke/container_smoke.py | 87 ++ .../tests/test_container_smoke.py | 126 ++ src/sp_validation/tests/test_csl_modules.py | 74 + uv.lock | 159 ++ workflow/README.md | 234 ++- workflow/Snakefile | 15 +- workflow/common.py | 107 ++ workflow/image_sims/Snakefile | 41 +- workflow/image_sims/config.yaml | 53 +- workflow/profiles/candide/config.yaml | 103 +- workflow/profiles/default/config.yaml | 39 + workflow/rules/covariance.smk | 40 +- workflow/rules/glass_mock.smk | 2 +- workflow/rules/image_sims.smk | 316 ++-- workflow/rules/twopoint.smk | 87 +- .../scripts/analyze_mask_power_spectrum.py | 2 - workflow/scripts/cosmocov_process.py | 73 + workflow/scripts/cv_additive_bias.py | 1 - workflow/scripts/cv_cosebis.py | 1 - workflow/scripts/cv_footprints.py | 1 - workflow/scripts/cv_objectwise_leakage.py | 1 - workflow/scripts/cv_plot_2pcf.py | 1 - workflow/scripts/cv_plot_rho_stats.py | 1 - workflow/scripts/cv_plot_tau_stats.py | 1 - workflow/scripts/cv_pseudo_cl.py | 1 - workflow/scripts/cv_pure_eb.py | 1 - workflow/scripts/cv_ratio_xi_sys_xi.py | 1 - workflow/scripts/cv_rho_tau_fits.py | 1 - workflow/scripts/cv_summarize_bmodes.py | 1 - workflow/scripts/cv_weights.py | 1 - .../generate_glass_mock_rhotau_samples.py | 73 +- workflow/scripts/im_mbias_config.py | 105 ++ workflow/scripts/process_mask.py | 2 - workflow/scripts/run_rho_tau.py | 2 - 62 files changed, 2032 insertions(+), 2001 deletions(-) create mode 100644 papers/bmodes/scripts/container_env.sh delete mode 100644 papers/bmodes/scripts/unblinding_ceremony.py create mode 100755 src/sp_validation/container.py create mode 100644 src/sp_validation/tests/data/container_smoke/Snakefile create mode 100644 src/sp_validation/tests/data/container_smoke/container_smoke.py create mode 100644 src/sp_validation/tests/test_container_smoke.py create mode 100644 src/sp_validation/tests/test_csl_modules.py create mode 100644 workflow/profiles/default/config.yaml create mode 100644 workflow/scripts/cosmocov_process.py create mode 100644 workflow/scripts/im_mbias_config.py diff --git a/.gitignore b/.gitignore index a1a0394b..9c07b05b 100644 --- a/.gitignore +++ b/.gitignore @@ -195,6 +195,10 @@ papers/cosmo_val/logs/ # Ignore scratch notebooks scratch/*/*.ipynb + +# Snakemake run state +.snakemake/ + # CI lint-gate scratch output (lint.yml writes these in the run tree; never commit) check.txt format.txt diff --git a/CLAUDE.md b/CLAUDE.md index e985d6c2..c35c20be 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,10 +78,26 @@ Main configuration in `scripts/calibration/params.py` with parameters: - pyccl for cosmological calculations ## Container Usage -Recommended installation via Apptainer/Docker: +Nothing is hand-built. CI publishes `ghcr.io/cosmostat/sp_validation:` on +every push, and each person keeps their own copy at +`~/.cache/sp_validation/sp_validation.sif`, managed by the `spv-container` CLI: + ```bash -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop +spv-container pull # fetch :develop there (do it from a compute node) +spv-container status # which layer is live, and how current it is +spv-container exec # one-off run inside it ``` +Need a package the image lacks mid-analysis? `spv-container sandbox`, then +`spv-container exec --writable pip install `; the sandbox then takes +precedence over the SIF everywhere, workflow jobs included. + +Every rule runs inside that image, wrapped by Snakemake itself (`--profile +workflow/profiles/candide` on the cluster, `workflow/profiles/default -j N` +elsewhere). The `sp_validation` a rule imports comes from the *launched +checkout*, not the image. + +`workflow/README.md` is the full story — profiles, image resolution, refresh. + ## Notebook Configuration - The CosmologyValidation class must be initialized in cosmo_val \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00a8d907..e385a386 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,15 +13,49 @@ inside the project container, which ships the full stack pre-built. ### Container (recommended) +CI builds and pushes an image on **every** push, tagged by the sanitized branch +name (see +[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so +`:develop` tracks the integration branch and your branch has an image of its +own. Nothing is built by hand. + +You keep your own copy of the image. `spv-container` — stdlib-only, so it runs +straight from a checkout: symlink it onto your PATH (`ln -s +"$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container`, the README's +install step) or call it as `python3 src/sp_validation/container.py` — pulls it to +`~/.cache/sp_validation/sp_validation.sif` and runs things inside it: + ```bash -# build a writeable sandbox from the published image -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop -apptainer shell --writable sp_validation +spv-container pull # fetch :develop; ~1.5 GB, so do it from a compute node +spv-container status # which layer is live, and how current it is +spv-container exec bash # an interactive shell inside it ``` -The image is rebuilt and pushed on every push to `develop` (see -[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so -`:develop` always tracks the latest integration branch. +That image is read-only. When you need a package it does not carry yet, unpack a +writable sandbox once with `spv-container sandbox` and install into it with +`spv-container exec --writable pip install `; the sandbox then takes +precedence everywhere, workflow jobs included. Treat it as an exploration tool — +the real fix is adding the dependency to `pyproject.toml` — and reset it with +`spv-container pull && spv-container sandbox --force`. + +Analysis runs through Snakemake, which wraps every job in `apptainer exec` +against that same image for you — see +[`workflow/README.md`](workflow/README.md) for the profiles and the details. + +Two things worth knowing while developing: + +- **Your checkout's code is what runs.** The workflow prepends the launched + checkout's `src/` to the container's `PYTHONPATH`, so the image supplies the + dependency stack and your working tree supplies `sp_validation`. No rebuild + needed to test a change. (Caveat: `rerun-triggers: code` does not watch + `src/`, so force reruns after editing a module.) +- **To test a branch's own image** — when the *stack* changed, not just `src/` — + point the workflow at its CI tag: + + ```bash + snakemake --profile workflow/profiles/candide \ + --config container=docker://ghcr.io/cosmostat/sp_validation:my-branch + ``` ### Local install with `uv` diff --git a/Dockerfile b/Dockerfile index ccc930bf..79fdf0d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ # Development image with more bells and whistles FROM ghcr.io/cosmostat/shapepipe:im_sims +# liblapack-dev: cosmosis's MultiNest links -llapack, and the base image ships +# only the runtime liblapack.so.3 (no dev symlink). The gsl/cfitsio/fftw3 dev +# packages are what the CosmoSIS Standard Library's C sources compile against +# (they are the headers CSL's own CI installs); git is for cloning it. RUN apt-get update -y --quiet --fix-missing && \ apt-get dist-upgrade -y --quiet --fix-missing && \ apt-get install -y --quiet \ @@ -8,14 +12,24 @@ RUN apt-get update -y --quiet --fix-missing && \ automake \ libtool \ pkg-config \ + git \ htop \ npm \ - tmux + tmux \ + liblapack-dev \ + libgsl-dev \ + libcfitsio-dev \ + libfftw3-dev # The base shapepipe image provides a uv-managed venv at /app/.venv (exported as # VIRTUAL_ENV); install sp_validation's deps into that same venv rather than # spawning a second one under /sp_validation. ENV UV_PROJECT_ENVIRONMENT=/app/.venv +# $HOME is bind-mounted under apptainer, so uv would otherwise discover the +# host's managed CPythons -- including newer ones that satisfy requires-python +# -- and build a venv against an interpreter carrying none of this stack. +ENV UV_PYTHON=/app/.venv/bin/python \ + UV_PYTHON_DOWNLOADS=never WORKDIR /sp_validation @@ -30,9 +44,58 @@ WORKDIR /sp_validation # numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake # and cs_util `--upgrade` layers are gone. COPY pyproject.toml uv.lock /sp_validation/ + +# cosmosis builds MPI-enabled polychord/multinest only when MPIFC is set: its +# setup.py exports MPIFC for conda builds only, and the sampler Makefiles gate on +# `which $(MPIFC)`. Absolute path, not a bare name: /opt/ompi/bin is not always on +# PATH, and a miss silently omits libchord_mpi.so while the install still succeeds, +# and `cosmosis --mpi` fails at load time -- which is how the pipeline runs, since +# the --smp pool is broken upstream. Must precede the sync that builds cosmosis. +ENV MPIFC=/opt/ompi/bin/mpif90 + RUN uv sync --frozen --inexact --no-install-project \ --extra test --extra glass --extra workflow +# The CosmoSIS Standard Library: the module files (camb interface, projection, +# 2pt likelihood, ...) the cosmo_inference pipelines name. The `workflow` extra +# above installs cosmosis itself; CSL is a separate tree of modules that is not +# on PyPI and has to be built against that install, so it is cloned and compiled +# here rather than left to each user (which is what the .ini templates used to +# assume, hard-coding one person's home directory). +# +# CSL_REF pins UNIONS-WL fork main: Sacha's four UNIONS commits reapplied +# on current upstream, including the scipy lpn fix. +ARG CSL_REPO=https://github.com/UNIONS-WL/cosmosis-standard-library.git +ARG CSL_REF=b7b1552a02ad9c39c9bb1e68e3f17213a8f740e1 +ENV CSL_DIR=/opt/cosmosis-standard-library + +# `python -m cosmosis.configure` emits the exports (COSMOSIS_SRC_DIR et al.) +# every CSL Makefile includes its compiler config from. Evaluated directly +# rather than through the `cosmosis-configure` wrapper: that wrapper's +# am-I-sourced probe reads unset zsh/ksh variables, which `set -u` turns into +# an error, and its `exit` then ends the sourcing shell with status 0 — make +# never runs and the layer still "succeeds". The trailing `test -f` keeps any +# such silent no-op loud. +# +# `make -C shear` rather than a bare `make`: the top-level target also descends +# into likelihood/, which builds the Planck, WMAP and ACT likelihoods -- large, +# data-dependent, and unused by any UNIONS pipeline. Everything our .ini +# templates reference is either pure Python (consistency, sample_S8, camb, +# load_nz_fits, photoz_bias, linear_alignment, add_intrinsic, shear_m_bias, +# xi_sys, 2pt_like -- no Makefile in those trees at all) or lives under shear/: +# `limber`, which project_2d.py links, and `cl_to_xi_nicaea`, whose +# nicaea_interface.so the 2pt_shear stage loads. +RUN bash -c 'set -eo pipefail; \ + export PATH=/app/.venv/bin:$PATH; \ + git clone --filter=blob:none "$CSL_REPO" "$CSL_DIR"; \ + cd "$CSL_DIR"; \ + git checkout --detach "$CSL_REF"; \ + cmds=$(python -m cosmosis.configure); \ + eval "$cmds"; \ + export GSL_INC=/usr/include GSL_LIB=/usr/lib/x86_64-linux-gnu; \ + make -C shear; \ + test -f shear/cl_to_xi_nicaea/nicaea_interface.so' + # Install sp_validation itself (editable) into the same venv; deps are already # satisfied by the sync above. COPY . /sp_validation diff --git a/README.md b/README.md index e14a2071..455ddd91 100644 --- a/README.md +++ b/README.md @@ -62,30 +62,35 @@ directive imports the shared rules under each run's own config and an output `prefix`, so runs namespace under `results//` without clobbering one another. -## Container Installation (Recommended) +## Installation -The easiest way to install sp_validation is via a container. Docker images are automatically built and pushed to the [GitHub Container Registry (GHCR)](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation) on every push to `develop`. This image can be installed and run on most systems (including clusters) with just a few lines of code. - -We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command: +`sp_validation` runs from a pre-built container: CI builds an image carrying +the full scientific stack on every push and publishes it to the +[GitHub Container Registry](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation). +The bundled `spv-container` CLI installs and manages your personal copy of it: ```bash -# build writeable "sandbox" container in the current directory -# ./sp_validation will be a directory that functions like a vm -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop - -# open a shell in the container -apptainer shell --writable sp_validation -# and confirm that the installation was successful -python -c "import sp_validation" -``` +git clone https://github.com/CosmoStat/sp_validation.git +cd sp_validation +ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container -You can also run the image with **Docker**: - -```bash -docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp_validation" +spv-container pull # fetch the image (~1.5 GB) +spv-container exec python -c "import sp_validation" # confirm it works ``` -We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance. +That is the whole install. `pull` puts the image at its canonical per-user +path (`~/.cache/sp_validation/`), and everything else finds it there — +`spv-container exec` for one-off commands (`spv-container exec bash` for an +interactive shell) and the Snakemake workflow for cluster jobs. +`spv-container status` says what you have and how current it is; +`spv-container sandbox` gives you a writable copy for mid-analysis +`pip install`s. On a cluster, run the pull from a compute node. + +To run the analysis workflow (`workflow/`), see +[`workflow/README.md`](workflow/README.md): Snakemake runs on the host, and +the profile puts each job in the container itself. For Docker, development +installs, and more depth, see the +[installation docs](https://cosmostat.github.io/sp_validation/installation.html). diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 5d753010..d45eb148 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -4,15 +4,43 @@ by Lisa Goh and Sacha Guerrini, CEA Paris-Saclay This folder contains the files neccessary to run the cosmological inference pipeline on the UNIONS galaxy catalogues. ### Requirements -To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/). To sample the PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. +Everything the pipeline needs ships in the container: nothing to install, and no +paths to edit before a run. + +[CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) comes in via the +`workflow` extra, built with MPI support. The CosmoSIS Standard Library — the +tree of modules the `.ini` pipelines name — is built into the image at +`/opt/cosmosis-standard-library`, with `CSL_DIR` pointing there. CosmoSIS reads +environment variables into an `.ini`'s `[DEFAULT]` section, so the templates' +`COSMOSIS_DIR = %(CSL_DIR)s` resolves to it. Outside the container, export +`CSL_DIR` at a build of your own and the same templates work unchanged. + +CSL is pinned to the **UNIONS-WL org fork** +([UNIONS-WL/cosmosis-standard-library](https://github.com/UNIONS-WL/cosmosis-standard-library/)) +at `b7b1552a`, the fork's main branch: Sacha's four UNIONS commits are +reapplied on current upstream, including the scipy `lpn` fix. + +Launch sampling under MPI (`mpiexec -n N cosmosis --mpi ...`), not `--smp`: +CosmoSIS's shared-memory pool is unmaintained and still crashes after sampling +completes (`Pool` has no attribute `data`, `runtime/process_pool.py`) as of +3.25.2. ### To Run -The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: +The inference pipeline is orchestrated through Snakemake. On the candide +cluster, drive it with the committed profile — see +[`workflow/README.md`](../workflow/README.md) for the one-time +`uv tool install` setup and the full explanation. From the repository root: ```bash -snakemake -j inference_fiducial +snakemake --profile workflow/profiles/candide \ + -s workflow/Snakefile \ + inference_fiducial --configfile ``` +Off-cluster, drop `--profile` and add `-j ` instead. Each job runs +inside the sp_validation container automatically — no `apptainer shell` or +`apptainer exec` needed by hand. + This will automatically execute all steps: 1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` 2. Compute covariance matrices using CosmoCov diff --git a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini index eb3ab166..656c6e5c 100644 --- a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini +++ b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini @@ -1,6 +1,8 @@ #parameters used elsewhere in this file [DEFAULT] -COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa +# The CosmoSIS Standard Library; CSL_DIR comes from the environment, set in the +# container (see cosmo_inference/README.md). +COSMOSIS_DIR = %(CSL_DIR)s [pipeline] diff --git a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini index 87f06064..4a365195 100644 --- a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini +++ b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini @@ -1,6 +1,8 @@ #parameters used elsewhere in this file [DEFAULT] -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library +# The CosmoSIS Standard Library; CSL_DIR comes from the environment, set in the +# container (see cosmo_inference/README.md). +COSMOSIS_DIR = %(CSL_DIR)s [pipeline] diff --git a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini index f9f4da51..341baf21 100644 --- a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini +++ b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini @@ -1,6 +1,8 @@ #parameters used elsewhere in this file [DEFAULT] -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library +# The CosmoSIS Standard Library; CSL_DIR comes from the environment, set in the +# container (see cosmo_inference/README.md). +COSMOSIS_DIR = %(CSL_DIR)s [pipeline] diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f84acebc..56de599f 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -2,28 +2,59 @@ Installation ============ ``sp_validation`` is **not** distributed on PyPI. -Install it from a pre-built container, or check out the source with ``uv`` when you need to edit it. +It runs from a pre-built container, managed by the bundled ``spv-container`` CLI; check out the source with ``uv`` only when you need to edit the package itself. -Container (recommended) ------------------------ +Container via ``spv-container`` (recommended) +--------------------------------------------- -Every push to ``develop`` builds an image carrying the full scientific stack and pushes it to the `GitHub Container Registry (GHCR) -`_. +Every push builds an image carrying the full scientific stack and pushes it to the `GitHub Container Registry (GHCR) +`_, tagged by branch — ``:develop`` tracks the integration branch. The image runs on most systems, including HPC clusters, with no further setup. +``spv-container`` installs your personal copy of it and manages it from then on: -`Apptainer `_ (formerly Singularity) is installed on most clusters and is the path we recommend: +.. code-block:: bash + + git clone https://github.com/CosmoStat/sp_validation.git + cd sp_validation + ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container + + spv-container pull # fetch the image (~1.5 GB) + spv-container exec python -c "import sp_validation" # confirm it works + +The symlink works because ``container.py`` is deliberately stdlib-only: it runs on the *host*, where the science stack is not installed. +(Inside the container the same CLI is on ``PATH`` as a console script.) +``pull`` requires `Apptainer `_ (formerly Singularity), which is installed on most clusters, and writes the image to one canonical per-user path, ``~/.cache/sp_validation/sp_validation.sif``. +Each user owns their copy: you refresh it when you want to, and nobody else's refresh moves the ground under your running jobs. +On a cluster, run the pull from a compute node — it moves ~1.5 GB. + +The subcommands: .. code-block:: bash - # Build a writeable "sandbox" container in the current directory. - # ./sp_validation is a directory that behaves like a small VM. - apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop + spv-container pull # fetch the published image to the canonical path + spv-container status # what is here, which commit built it, how current + spv-container exec # run a command inside it (exec bash for a shell) + spv-container sandbox # unpack into a writable dir, for pip installs + spv-container exec --writable # ... with writes that persist + +``status`` compares the image's build commit against your checkout's ``HEAD``, so you always know whether a ``pull`` would refresh anything. +The **sandbox** is the escape hatch for exploratory work that needs a package the image does not carry yet: once built, it takes precedence over the SIF everywhere — Snakemake workflow jobs included — until you reset with ``spv-container pull`` + ``spv-container sandbox --force``. + +Everything resolves the image in one order — sandbox if it exists, else your SIF, else the registry tag — and that includes the analysis workflow. +How the workflow uses the image (Snakemake runs on the host; the profile puts each job in the container) is covered in ``workflow/README.md``. + +Other ways to run the image +--------------------------- + +The published image is a normal OCI image; ``spv-container`` is a convenience, not a gatekeeper. +Run it directly with Apptainer: + +.. code-block:: bash - # Open a shell in the container, then confirm the install works. - apptainer shell --writable sp_validation - python -c "import sp_validation" + apptainer pull sp_validation.sif docker://ghcr.io/cosmostat/sp_validation:develop + apptainer shell sp_validation.sif -The image also runs under Docker: +or with Docker: .. code-block:: bash diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index 0adf66a1..c58663d6 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -4,8 +4,6 @@ configfile: "config/config.yaml" configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" -container: "/n17data/cdaley/containers/containers" - envvars: "PYTHONUNBUFFERED", @@ -27,6 +25,9 @@ import common common.configure(config) from common import * +# The one image every rule runs in; see common.resolve_container. +container: common.resolve_container(config.get("container")) + # Wildcard constraints — centralized in common.py, not in individual rule files wildcard_constraints: **WILDCARD_CONSTRAINTS diff --git a/papers/bmodes/rules/presentation.smk b/papers/bmodes/rules/presentation.smk index a9ef5dc1..c59f0aa3 100644 --- a/papers/bmodes/rules/presentation.smk +++ b/papers/bmodes/rules/presentation.smk @@ -59,7 +59,11 @@ rule talk_previews: rule talk_figure: - """Convert a single paper PDF to high-res PNG for the talk.""" + """Convert a single paper PDF to high-res PNG for the talk. + + `container: None` on purpose: ImageMagick's `convert` is a host tool and is + not installed in the sp_validation image. + """ input: pdf=lambda w: TALK_FIGURES[w.name], output: @@ -73,7 +77,11 @@ rule talk_figure: rule talk_figure_preview: - """Downscale a talk figure to < 1800px for safe AI reading.""" + """Downscale a talk figure to < 1800px for safe AI reading. + + `container: None` for the same reason as talk_figure: `convert` is a host + tool, absent from the image. + """ input: f"{TALK_DIR}/images/{{name}}.png", output: @@ -120,8 +128,6 @@ rule presentation_omega_m_difference: mock_summary="/n09data/guerrini/glass_mock_chains/summary_parameter_constraints_merged_v6.txt", output: f"{TALK_DIR}/images/omega_m_difference_config_harm.png", - container: - None shell: "python {TALK_DIR}/plot_omega_m_difference.py" @@ -132,8 +138,6 @@ rule presentation_s8_scatter_mocks: mock_summary="/n09data/guerrini/glass_mock_chains/summary_parameter_constraints_merged_v6.txt", output: f"{TALK_DIR}/images/s8_scatter_config_vs_harmonic.png", - container: - None shell: "python {TALK_DIR}/plot_s8_scatter_mocks.py" @@ -147,11 +151,5 @@ rule presentation_pte_cosebis: ], output: f"{TALK_DIR}/images/pte_cosebis_talk.png", - container: - None shell: - """ - apptainer exec --bind /home,/scratch,/automnt,/n17data,/n23data1,/n09data \ - /n17data/cdaley/containers/containers/ \ - python {TALK_DIR}/plot_pte_cosebis_talk.py - """ + "python {TALK_DIR}/plot_pte_cosebis_talk.py" diff --git a/papers/bmodes/rules/synthesis.smk b/papers/bmodes/rules/synthesis.smk index fb9208e1..64b60770 100644 --- a/papers/bmodes/rules/synthesis.smk +++ b/papers/bmodes/rules/synthesis.smk @@ -39,7 +39,7 @@ def _claim_outputs(): # Paper Macros # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -localrules: xi_cosmology_paper, paper_macros, bmodes_paper_spec, all_tapestry, unblinding_ceremony +localrules: xi_cosmology_paper, paper_macros, bmodes_paper_spec, all_tapestry rule xi_cosmology_paper: """Spec for B-mode reporting in configuration-space paper (Goh et al.). @@ -128,40 +128,6 @@ rule bmodes_paper_spec: # Aggregate Targets # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -_CEREMONY_BLIND = config.get("ceremony_blind", "A") -_CEREMONY_CHAIN_ROOT = "/n09data/guerrini/output_chains" -_CEREMONY_COSMOSIS_DIR = "/home/guerrini/sp_validation/cosmo_inference/data" - -rule unblinding_ceremony: - """Generate unblinding ceremony figure sequence. - - Via snakemake: - snakemake unblinding_ceremony --config ceremony_blind=B --nolock - Standalone: - app python workflow/scripts/unblinding_ceremony.py B - """ - input: - xi_data=f"{_CEREMONY_COSMOSIS_DIR}/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}/cosmosis_{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}.fits", - pure_eb=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_pure_eb_semianalytic.npz", - pseudo_cl=_pseudo_cl_path(FIDUCIAL_VERSION, blind=_CEREMONY_BLIND), - pseudo_cl_cov=_pseudo_cl_cov_path(FIDUCIAL_VERSION, blind=_CEREMONY_BLIND), - cosmosis_cell_fits=f"{_CEREMONY_COSMOSIS_DIR}/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_fid/cosmosis_{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_fid_cell.fits", - bestfit_dir=f"{_CEREMONY_CHAIN_ROOT}/best_fit/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_10_80/shear_xi_plus/theta.txt", - output: - evidence=f"{TAPESTRY_DIR}/unblinding_ceremony/evidence.json", - params: - blind=_CEREMONY_BLIND, - chain_version=FIDUCIAL_VERSION.replace("SP_", "").replace("_leak_corr", ""), - chain_prefix=FIDUCIAL_VERSION, - chain_root_dir=_CEREMONY_CHAIN_ROOT, - results_dir="results/unblinding", - bestfit_root_fid_cell=f"{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_fid_cell", - bestfit_root_halofit_cell=f"{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_halofit_cell", - bestfit_root_config=f"{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_10_80", - shell: - "python workflow/scripts/unblinding_ceremony.py {params.blind} --chain-version {params.chain_version}" - - rule all_tapestry: """Aggregate target for all claim evidence and paper outputs.""" input: diff --git a/papers/bmodes/scripts/container_env.sh b/papers/bmodes/scripts/container_env.sh new file mode 100644 index 00000000..a63b40f5 --- /dev/null +++ b/papers/bmodes/scripts/container_env.sh @@ -0,0 +1,45 @@ +# Shared environment for the run_*.sh sweep drivers. Source, don't run: +# +# . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" +# +# Sets the checkout the drivers run out of, the container to run in, and +# `spv_python`, which is how every driver invokes python inside it. +WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra +SRC=$WT/src +WSCRIPTS=$WT/workflow/scripts +PSCRIPTS=$WT/papers/bmodes/scripts + +# CONTAINER and BIND are resolved exactly as `sp_validation/container.py` does: +# the writable sandbox if there is one, else the SIF. +_spv_cache=${XDG_CACHE_HOME:-$HOME/.cache}/sp_validation +_spv_sandbox=${SPV_SANDBOX:-$_spv_cache/sandbox} +if [ -d "$_spv_sandbox" ]; then + CONTAINER=$_spv_sandbox +else + CONTAINER=${SPV_CONTAINER:-$_spv_cache/sp_validation.sif} +fi +BIND=${SPV_APPTAINER_BINDS:-/home,/scratch,/automnt,/n17data,/n23data1,/n09data} +unset _spv_cache _spv_sandbox + +# Every math library pinned to one thread -- pass as SPV_EXEC_EXTRA where the +# parallelism is by process, not by thread. +SINGLE_THREAD_ENV="--env OMP_NUM_THREADS=1 --env OPENBLAS_NUM_THREADS=1 + --env MKL_NUM_THREADS=1 --env NUMBA_NUM_THREADS=1 --env NUMEXPR_NUM_THREADS=1 + --env VECLIB_MAXIMUM_THREADS=1" + +# Run python inside the container against the checkout's src. Extra +# `apptainer exec` flags go in SPV_EXEC_EXTRA (word-split on purpose). +spv_python() { + apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" ${SPV_EXEC_EXTRA:-} \ + "$CONTAINER" /usr/local/bin/python "$@" +} + +# Echo the version list a sweep runs over: $VERSIONS if the caller set one, +# else whatever sweep_versions.py resolves from $1 (a config path). +sweep_versions() { + if [ -n "${VERSIONS:-}" ]; then + echo "$VERSIONS" + else + spv_python "$PSCRIPTS/sweep_versions.py" --config "$1" + fi +} diff --git a/papers/bmodes/scripts/filter_catalog_ellipticity.py b/papers/bmodes/scripts/filter_catalog_ellipticity.py index ade5dd6d..baab3ffb 100644 --- a/papers/bmodes/scripts/filter_catalog_ellipticity.py +++ b/papers/bmodes/scripts/filter_catalog_ellipticity.py @@ -18,7 +18,6 @@ sys.stderr if hasattr(sys, "ps1") else open(sys.stderr.fileno(), "w", buffering=1) ) -from snakemake.script import snakemake # noqa: E402 input_path = snakemake.input["catalog"] output_fits = snakemake.output["catalog"] diff --git a/papers/bmodes/scripts/plot_pure_eb_covariance.py b/papers/bmodes/scripts/plot_pure_eb_covariance.py index 3fdba8db..5522ef59 100644 --- a/papers/bmodes/scripts/plot_pure_eb_covariance.py +++ b/papers/bmodes/scripts/plot_pure_eb_covariance.py @@ -27,8 +27,6 @@ def _load_snakemake(): "results/paper_plots/pure_eb_covariance.png", str(Path.cwd()), ) - from snakemake.script import snakemake - return snakemake diff --git a/papers/bmodes/scripts/run_cl_sweep.py b/papers/bmodes/scripts/run_cl_sweep.py index 8cb2106e..d5fd6eb0 100644 --- a/papers/bmodes/scripts/run_cl_sweep.py +++ b/papers/bmodes/scripts/run_cl_sweep.py @@ -13,7 +13,7 @@ ``mask:`` entry), so no per-version mask wiring is needed here. Serial over versions; the estimator uses the recipe's full core allocation per call. - apptainer exec ... /usr/local/bin/python run_cl_sweep.py \ + python run_cl_sweep.py \ --config .../config.yaml --cat-config .../cat_config.yaml \ --nside 1024 --npatch 1 --binning powspace --nbins 32 --power 0.5 \ --blind A --out diff --git a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py index 26b3d996..df3b9b79 100644 --- a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py +++ b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py @@ -14,7 +14,7 @@ config_space_pte_matrices.py adapts via ``_cosebis_matrix_from_npz`` — into ``--out``. Serial over versions (~25 min/version, 206 pairs). - apptainer exec ... /usr/local/bin/python run_cosebis_ptes_sweep.py \ + python run_cosebis_ptes_sweep.py \ --config .../config.yaml \ --xi-sweep-dir \ --cov-sweep-dir \ diff --git a/papers/bmodes/scripts/run_cov_sweep.sh b/papers/bmodes/scripts/run_cov_sweep.sh index 88d7f593..f86f20dc 100755 --- a/papers/bmodes/scripts/run_cov_sweep.sh +++ b/papers/bmodes/scripts/run_cov_sweep.sh @@ -26,12 +26,7 @@ # [--blind A] [--versions "v1 v2 ..."] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" CONFIG=""; CATCONFIG=""; PLANCK18=""; MASKBASE=""; OUT=""; BLIND="A"; VERSIONS="" MINSEP=0.5; MAXSEP=300.0; NBINS=1000 @@ -50,10 +45,7 @@ done mkdir -p "$OUT" -if [ -z "$VERSIONS" ]; then - VERSIONS=$(apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/sweep_versions.py" --config "$CONFIG") -fi +VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do base="covariance_${ver}_${BLIND}_g_minsep=${MINSEP}_maxsep=${MAXSEP}_nbins=${NBINS}_masked" diff --git a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh index 34ba5f7b..0ea3a862 100644 --- a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh @@ -19,12 +19,7 @@ # --out [--blind A] [--versions "v1 v2 ..."] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" while [ $# -gt 0 ]; do @@ -42,10 +37,7 @@ done mkdir -p "$OUT" -if [ -z "$VERSIONS" ]; then - VERSIONS=$(apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/sweep_versions.py" --config "$CONFIG") -fi +VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do pureeb="$PUREEBSWEEP/${ver}_${BLIND}_pure_eb_semianalytic.npz" @@ -55,8 +47,7 @@ for ver in $VERSIONS; do [ -f "$f" ] || { echo "MISSING upstream input for $ver: $f" >&2; exit 1; } done echo "[pure_eb_ptes_sweep] $ver" - apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ + spv_python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ --version "$ver" --blind "$BLIND" \ --pure-eb-data "$pureeb" --cov-integration "$covint" \ --npatch 1 --n-samples 2000 --out "$OUT" diff --git a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh index 609f2a18..472cd81d 100644 --- a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh +++ b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh @@ -14,10 +14,7 @@ # --out [--n-chunks 20] [--n-samples 2000] [--nproc 16] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -SRC=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra/src -SCRIPTS=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" VERSION=""; BLIND="A"; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" NCHUNKS=20; NSAMPLES=2000; NPROC="${SLURM_CPUS_PER_TASK:-16}" @@ -52,11 +49,8 @@ mkdir -p "$OUT/chunks" echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION blind=$BLIND" for i in $(seq 0 $((NCHUNKS-1))); do ( - apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" \ - --env OMP_NUM_THREADS=1 --env OPENBLAS_NUM_THREADS=1 --env MKL_NUM_THREADS=1 \ - --env NUMBA_NUM_THREADS=1 --env NUMEXPR_NUM_THREADS=1 --env VECLIB_MAXIMUM_THREADS=1 \ - "$CONTAINER" \ - /usr/local/bin/python "$SCRIPTS/precompute_pure_eb_chunk.py" \ + SPV_EXEC_EXTRA=$SINGLE_THREAD_ENV + spv_python "$PSCRIPTS/precompute_pure_eb_chunk.py" \ --chunk-id "$i" --n-chunks "$NCHUNKS" --n-samples "$NSAMPLES" \ --version "$VERSION" --blind "$BLIND" --cat-config "$CATCONFIG" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" --cov-integration "$COVINT" \ @@ -76,8 +70,7 @@ done [ "$missing" -eq 0 ] || { echo "[pure_eb] chunk failures — aborting gather" >&2; exit 1; } echo "[pure_eb] all $NCHUNKS chunks done; gathering" -apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$SCRIPTS/gather_pure_eb_chunks.py" \ +spv_python "$PSCRIPTS/gather_pure_eb_chunks.py" \ --version "$VERSION" --blind "$BLIND" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" \ --chunks-dir "$OUT/chunks" \ diff --git a/papers/bmodes/scripts/run_pure_eb_sweep.sh b/papers/bmodes/scripts/run_pure_eb_sweep.sh index 74b6dab3..8195f150 100755 --- a/papers/bmodes/scripts/run_pure_eb_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_sweep.sh @@ -19,12 +19,7 @@ # --out [--blind A] [--versions "v1 v2 ..."] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" while [ $# -gt 0 ]; do @@ -42,10 +37,7 @@ done mkdir -p "$OUT" -if [ -z "$VERSIONS" ]; then - VERSIONS=$(apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/sweep_versions.py" --config "$CONFIG") -fi +VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt" diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index b02283a5..dd7219a3 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -13,7 +13,7 @@ binning. Serial over versions — lc's dask handles cross-output concurrency, and TreeCorr already uses the recipe's full OpenMP allocation per call. - apptainer exec ... /usr/local/bin/python run_xi_sweep.py \ + python run_xi_sweep.py \ --config .../config.yaml --cat-config .../cat_config.yaml --out """ diff --git a/papers/bmodes/scripts/unblinding_ceremony.py b/papers/bmodes/scripts/unblinding_ceremony.py deleted file mode 100644 index 61ab3a95..00000000 --- a/papers/bmodes/scripts/unblinding_ceremony.py +++ /dev/null @@ -1,1389 +0,0 @@ -#!/usr/bin/env python3 -"""Run the UNIONS unblinding ceremony figure sequence. - -Usage (standalone) ------------------- -app python workflow/scripts/unblinding_ceremony.py A -app python workflow/scripts/unblinding_ceremony.py A --output-dir /path/to/output - -Usage (snakemake) ------------------ -snakemake unblinding_ceremony --config ceremony_blind=A -""" - -import argparse -import json -import shutil -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path - -_SCRIPT_DIR = Path(__file__).resolve().parent -if str(_SCRIPT_DIR) not in sys.path: - sys.path.insert(0, str(_SCRIPT_DIR)) - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits -from getdist import plots -from matplotlib import scale as mscale -from matplotlib.gridspec import GridSpec -from plotting_utils import PAPER_MPLSTYLE, SquareRootScale -from scipy.interpolate import interp1d - -# ── Plotting environment ──────────────────────────────────────────────────── -mscale.register_scale(SquareRootScale) -plt.style.use(PAPER_MPLSTYLE) -plt.rc("text", usetex=True) - - -# ── Data types ────────────────────────────────────────────────────────────── - - -@dataclass(frozen=True) -class ChainSpec: - root: str - label: str - color: str - base_dir: Path - alpha: float = 1.0 - - -FULL_PARAMS = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", -] -COSMO_PARAMS = ["OMEGA_M", "s_8_input", "SIGMA_8", "a"] - -MUTED_ALPHA = 0.25 - - -@dataclass -class CeremonyConfig: - """All paths and parameters needed by the ceremony, sourced from snakemake or CLI.""" - - blind: str - chain_version: str - chain_prefix: str - chain_root_dir: Path - external_root_dir: Path - results_dir: Path - evidence_dir: Path - - xi_data_path: Path - pure_eb_path: Path - pseudo_cl_path: Path - pseudo_cl_cov_path: Path - cosmosis_cell_fits: Path - bestfit_dir: Path - bestfit_root_fid_cell: str - bestfit_root_halofit_cell: str - bestfit_root_config: str - - -def _save_path(cfg: CeremonyConfig, index: int, slug: str) -> Path: - return cfg.results_dir / f"{index:02d}_{slug}.pdf" - - -# ── Chain loading (extracted from Sasha's notebooks) ──────────────────────── - - -def _load_xi_table(path: Path | str, nrows: int = 20) -> np.ndarray: - rows: list[np.ndarray] = [] - with Path(path).open("r", encoding="utf-8") as handle: - for line in handle: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - values = np.fromstring(stripped, sep=" ") - if values.size < 9: - continue - rows.append(values) - if len(rows) >= nrows: - break - if len(rows) < nrows: - raise ValueError( - f"Expected at least {nrows} xi rows in {path}, found {len(rows)}" - ) - return np.vstack(rows) - - -def ensure_getdist_chain(base_dir: Path, root: str) -> Path: - """MAKE PARAMNAMES FILE + READ CHAIN conversion (from notebooks).""" - chain_dir = base_dir / root - samples_path = chain_dir / f"samples_{root}.txt" - gd_samples_path = chain_dir / f"getdist_{root}.txt" - paramnames_path = chain_dir / f"getdist_{root}.paramnames" - - if not samples_path.exists(): - return gd_samples_path - - with samples_path.open("r", encoding="utf-8") as file: - params = file.readline()[1:].split("\t")[:-4] - - with paramnames_path.open("w", encoding="utf-8") as file: - for param in params: - if len(param.split("--")) > 1: - file.write(param.split("--")[1] + "\n") - else: - file.write(param.split("--")[0] + "\n") - - samples = np.loadtxt(samples_path) - if "nautilus" in root: - samples = np.column_stack( - (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3]) - ) - else: - samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4])) - np.savetxt(gd_samples_path, samples) - return gd_samples_path - - -def _build_plotter( - width_inch: float, - axes_fontsize: float, - axes_labelsize: float, - legend_fontsize: float, -): - g = plots.get_subplot_plotter(width_inch=width_inch) - g.settings.axes_fontsize = axes_fontsize - g.settings.axes_labelsize = axes_labelsize - g.settings.alpha_filled_add = 0.7 - g.settings.legend_fontsize = legend_fontsize - return g - - -def _set_param_labels(chain) -> None: - name_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "S_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", - ] - label_list = [ - r"\Omega_{\rm m}", - r"\omega_{\rm b} h^2", - r"h_0", - r"n_{\rm s}", - r"\sigma_8", - r"S_8", - r"S_8", - r"\log T_{\rm AGN}", - r"A_{\rm IA}", - r"m_1", - r"\Delta z_1", - ] - - param_names = chain.getParamNames() - for name, label in zip(name_list, label_list): - try: - param_names.parWithName(name).label = label - except Exception: - pass - - try: - param_names.parWithName("S_8") - except Exception: - try: - s8_input = chain.getParams().s_8_input - chain.addDerived(s8_input, name="S_8", label=r"S_8") - except Exception: - pass - - -def _adjust_paramname_chain( - chain, current_name: str, target_name: str, label: str -) -> None: - try: - param_names = chain.getParamNames() - par = param_names.parWithName(current_name) - par.label = label - par.name = target_name - chain.setParamNames(param_names) - except Exception: - pass - - -def _derive_parameter_s8(chain): - if "S_8" in chain.getParamNames().list(): - return chain - omega_m = chain.getParams().OMEGA_M - sigma_8 = chain.getParams().SIGMA_8 - s_8 = sigma_8 * (omega_m / 0.3) ** 0.5 - chain.addDerived(s_8, name="S_8", label=r"S_8") - return chain - - -def _harmonize_external_chain(chain, root: str) -> None: - if root in {"Planck18", "KiDS-1000", "HSC_Y3", "HSC_Y3_cell", "DES+KiDS", "DES_Y3"}: - _adjust_paramname_chain(chain, "omega_m", "OMEGA_M", r"\Omega_{\rm m}") - if root == "DES_Y3": - _derive_parameter_s8(chain) - - -def load_getdist_chains( - chain_specs: list[ChainSpec], - width_inch: float, - axes_fontsize: float, - axes_labelsize: float, - legend_fontsize: float, -): - g = _build_plotter( - width_inch=width_inch, - axes_fontsize=axes_fontsize, - axes_labelsize=axes_labelsize, - legend_fontsize=legend_fontsize, - ) - - chains = [] - for spec in chain_specs: - ensure_getdist_chain(spec.base_dir, spec.root) - chain = g.samples_for_root( - str(spec.base_dir / spec.root / f"getdist_{spec.root}"), - cache=False, - settings={"ignore_rows": 0, "smooth_scale_2D": 0.5, "smooth_scale_1D": 0.5}, - ) - _set_param_labels(chain) - if spec.base_dir.name == "ext_data" or spec.root in { - "Planck18", - "DES_Y3", - "KiDS-1000", - "DES+KiDS", - "HSC_Y3", - "HSC_Y3_cell", - }: - _harmonize_external_chain(chain, spec.root) - chains.append(chain) - - return g, chains - - -# ── Plot functions (extracted from Sasha's notebooks) ─────────────────────── - - -def plot_triangle( - chain_specs: list[ChainSpec], - param_names: list[str], - output_path: Path, - width_inch: float = 20.0, - axes_fontsize: float = 35.0, - axes_labelsize: float = 50.0, - legend_fontsize: float = 40.0, - legend_loc: str = "upper right", -) -> None: - """Extracted triangle_plot pattern from contour notebooks.""" - g, chains = load_getdist_chains( - chain_specs, width_inch, axes_fontsize, axes_labelsize, legend_fontsize - ) - - colours = [spec.color for spec in chain_specs] - linestyle = ["solid" for _ in chain_specs] - line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)] - - g.triangle_plot( - chains, - param_names, - legend_labels=[spec.label for spec in chain_specs], - line_args=line_args, - contour_colors=colours, - legend_loc=legend_loc, - filled=True, - ) - - output_path.parent.mkdir(parents=True, exist_ok=True) - g.export(str(output_path)) - plt.close(g.fig) - - -def plot_xipm_data_vector(xipm_path: str, output_path: Path) -> None: - """Extracted from 2D_cosmic_shear_paper_plots/corr_func.ipynb (xi+ / xi- blocks).""" - xipm = _load_xi_table(xipm_path, nrows=20) - theta = xipm[:, 1] - xip = xipm[:, 3] - xim = xipm[:, 4] - varxip = xipm[:, 7] - varxim = xipm[:, 8] - - fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, figsize=(10, 4.5)) - - ax1.tick_params( - axis="both", - which="both", - direction="in", - length=6, - width=1, - top=True, - bottom=True, - left=True, - right=True, - ) - ax1.yaxis.minorticks_on() - ax1.plot( - theta, - xip * 1e4, - marker="o", - markersize=4, - ls="solid", - lw=1.8, - color="royalblue", - ) - ax1.fill_between( - theta, (xip - varxip) * 1e4, (xip + varxip) * 1e4, color="powderblue", alpha=0.7 - ) - ax1.text( - 0.85, - 0.88, - "1-1", - transform=ax1.transAxes, - bbox=dict(facecolor="white", edgecolor="black", boxstyle="round", pad=0.5), - ) - ax1.axvspan(0, 10, color="gray", alpha=0.3) - ax1.axvspan(150, 200, color="gray", alpha=0.3) - ax1.set_xscale("log") - ax1.set_xlabel(r"$\theta$ [arcmin]") - ax1.set_ylabel(r"$\xi_+\times 10^4$") - - ax2.tick_params( - axis="both", - which="both", - direction="in", - length=6, - width=1, - top=True, - bottom=True, - left=True, - right=True, - ) - ax2.yaxis.minorticks_on() - ax2.plot( - theta, - xim * 1e4, - marker="o", - markersize=4, - ls="solid", - lw=1.8, - color="orangered", - ) - ax2.fill_between( - theta, (xim - varxim) * 1e4, (xim + varxim) * 1e4, color="pink", alpha=0.7 - ) - ax2.text( - 0.85, - 0.88, - "1-1", - transform=ax2.transAxes, - bbox=dict(facecolor="white", edgecolor="black", boxstyle="round", pad=0.5), - ) - ax2.axvspan(0, 10, color="gray", alpha=0.3) - ax2.axvspan(150, 200, color="gray", alpha=0.3) - ax2.set_xscale("log") - ax2.set_xlabel(r"$\theta$ [arcmin]") - ax2.set_ylabel(r"$\xi_-\times 10^4$") - - fig.tight_layout() - output_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -def plot_cell_ee_data_vector( - pseudo_cl_path: str, pseudo_cl_cov_path: str, output_path: Path -) -> None: - """Extracted from 2025_10_08_plot_data_vectors.py (EE panel logic).""" - cell = fits.getdata(pseudo_cl_path) - cov_cell = fits.open(pseudo_cl_cov_path) - - ell = cell["ell"] - cl_ee = cell["EE"] - cov_cl_ee = cov_cell["COVAR_EE_EE"].data - cov_cell.close() - - fig, ax0 = plt.subplots(ncols=1, nrows=1, figsize=(7, 5)) - ax0.errorbar( - ell, - cl_ee * ell, - yerr=np.sqrt(np.diag(cov_cl_ee)) * ell, - label=r"$C_\ell^{EE}$", - color="royalblue", - fmt="o", - capsize=2, - ) - - ax0.set_xscale("squareroot") - ax0.set_xticks(np.array([100, 400, 900, 1600])) - ax0.minorticks_on() - ax0.tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax0.set_xticks(minor_ticks, minor=True) - ax0.legend() - - ax0.set_xlabel(r"$\ell$") - ax0.set_ylabel(r"$\ell \, C_\ell^{EE}$") - - plt.tight_layout() - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -def plot_xipm_bestfit_with_bmodes( - xi_data_path: Path | str, - pure_eb_data_path: Path, - bestfit_dir: Path | None, - output_path: Path, - scale_cut_xip: tuple[float, float] = (12.0, 83.0), - scale_cut_xim: tuple[float, float] = (12.0, 83.0), -) -> None: - """Extracted from 2D_cosmic_shear_paper_plots/workflow/scripts/plot_xi_bestfit.py. - - If bestfit_dir is None, plots data + B-modes only (no theory curves). - - xi_data_path can be: - - A CosmoSIS FITS file with XI_PLUS/XI_MINUS HDUs and COVMAT (preferred) - - A plain-text TreeCorr output table (legacy) - """ - xi_path = Path(xi_data_path) - if xi_path.suffix == ".fits": - xip_hdu = fits.getdata(str(xi_path), "XI_PLUS") - xim_hdu = fits.getdata(str(xi_path), "XI_MINUS") - cov = fits.getdata(str(xi_path), "COVMAT") - theta_data = xip_hdu["ANG"] - xip_data = xip_hdu["VALUE"] - xim_data = xim_hdu["VALUE"] - n = len(xip_data) - sigma_xip = np.sqrt(np.diag(cov[:n, :n])) - sigma_xim = np.sqrt(np.diag(cov[n : 2 * n, n : 2 * n])) - else: - data = _load_xi_table(xi_path, nrows=20) - theta_data = data[:, 1] - xip_data = data[:, 3] - xim_data = data[:, 4] - sigma_xip = data[:, 7] - sigma_xim = data[:, 8] - - eb_data = np.load(pure_eb_data_path) - theta_eb = eb_data["theta"] - xip_B = eb_data["xip_B"] - xim_B = eb_data["xim_B"] - cov_pure_eb = eb_data["cov_pure_eb"] - - nbins = len(theta_eb) - sigma_xip_B = np.sqrt( - np.diag(cov_pure_eb[2 * nbins : 3 * nbins, 2 * nbins : 3 * nbins]) - ) - sigma_xim_B = np.sqrt( - np.diag(cov_pure_eb[3 * nbins : 4 * nbins, 3 * nbins : 4 * nbins]) - ) - - min_sep, max_sep = 1.0, 250.0 - bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) - bin_centers_nominal = np.sqrt(bin_edges[:-1] * bin_edges[1:]) - - def get_bin_edge_cuts(centers, edges, scale_cut): - mask = (centers >= scale_cut[0]) & (centers <= scale_cut[1]) - idx_first = np.where(mask)[0][0] - idx_last = np.where(mask)[0][-1] - return edges[idx_first], edges[idx_last + 1] - - edge_cut_xip = get_bin_edge_cuts(bin_centers_nominal, bin_edges, scale_cut_xip) - edge_cut_xim = get_bin_edge_cuts(bin_centers_nominal, bin_edges, scale_cut_xim) - - has_theory = bestfit_dir is not None - if has_theory: - theta_theory_rad = np.loadtxt( - bestfit_dir / "shear_xi_plus" / "theta.txt", comments="#" - ) - theta_theory = np.rad2deg(theta_theory_rad) * 60 - xip_theory = np.loadtxt( - bestfit_dir / "shear_xi_plus" / "bin_1_1.txt", comments="#" - ) - xim_theory = np.loadtxt( - bestfit_dir / "shear_xi_minus" / "bin_1_1.txt", comments="#" - ) - - theta_sys_rad = np.loadtxt(bestfit_dir / "xi_sys" / "theta.txt", comments="#") - theta_sys = np.rad2deg(theta_sys_rad) * 60 - xip_sys = np.loadtxt(bestfit_dir / "xi_sys" / "shear_xi_plus.txt", comments="#") - xim_sys = np.loadtxt( - bestfit_dir / "xi_sys" / "shear_xi_minus.txt", comments="#" - ) - - theta_fine = np.geomspace(0.5, 300, 500) - if has_theory: - xip_th_interp = interp1d( - theta_theory, xip_theory, kind="cubic", fill_value="extrapolate" - )(theta_fine) - xim_th_interp = interp1d( - theta_theory, xim_theory, kind="cubic", fill_value="extrapolate" - )(theta_fine) - xip_sys_interp = interp1d( - theta_sys, xip_sys, kind="cubic", fill_value="extrapolate" - )(theta_fine) - xim_sys_interp = interp1d( - theta_sys, xim_sys, kind="cubic", fill_value="extrapolate" - )(theta_fine) - - scale_factor = 1e-4 - xlim = [1, 250] - ylim = [-0.15, 1.25] - - ms_data = 3 - ms_bmode = 3 - capsize = 1.5 - elinewidth = 0.8 - - fig, axes = plt.subplots(1, 2, figsize=(10, 4.5), sharey=True) - - plot_configs = [ - ( - axes[0], - xip_data, - sigma_xip, - xip_B, - sigma_xip_B, - xip_th_interp if has_theory else None, - xip_sys_interp if has_theory else None, - edge_cut_xip, - r"$\xi_+$", - "+", - ), - ( - axes[1], - xim_data, - sigma_xim, - xim_B, - sigma_xim_B, - xim_th_interp if has_theory else None, - xim_sys_interp if has_theory else None, - edge_cut_xim, - r"$\xi_-$", - "-", - ), - ] - - for idx, ( - ax, - xi_data_arr, - sigma_xi, - xi_B, - sigma_B, - xi_th, - xi_sys_arr, - edge_cut, - label, - _pm, - ) in enumerate(plot_configs): - show_legend = idx == 1 - - ax.axvspan(xlim[0], edge_cut[0], color="0.90", zorder=0, alpha=0.7) - ax.axvspan(edge_cut[1], xlim[1], color="0.90", zorder=0, alpha=0.7) - - if has_theory: - ax.plot( - theta_fine, - theta_fine * (xi_th + xi_sys_arr) / scale_factor, - "-", - color="k", - lw=1.5, - label=r"Best-fit $\xi^{\mathrm{th}}_\pm + \xi^{\mathrm{sys}}_\pm$" - if show_legend - else None, - zorder=2, - ) - ax.plot( - theta_fine, - theta_fine * xi_sys_arr / scale_factor, - "-", - color="C0", - lw=1.2, - label=r"Best-fit $\xi^{\mathrm{sys}}_\pm$" if show_legend else None, - zorder=2, - ) - - ax.errorbar( - theta_data, - theta_data * xi_data_arr / scale_factor, - yerr=theta_data * sigma_xi / scale_factor, - fmt="o", - color="k", - markersize=ms_data, - capsize=capsize, - elinewidth=elinewidth, - label=r"$\xi_\pm$" if show_legend else None, - zorder=3, - ) - - theta_eb_offset = theta_eb * 1.03 - ax.errorbar( - theta_eb_offset, - theta_eb_offset * xi_B / scale_factor, - yerr=theta_eb_offset * sigma_B / scale_factor, - fmt="o", - color="C3", - markersize=ms_bmode, - capsize=capsize, - elinewidth=elinewidth, - alpha=0.85, - label=r"$\xi^B_\pm$" if show_legend else None, - zorder=3, - ) - - ax.axhline(0, color="gray", linestyle="--", alpha=0.8, linewidth=0.8, zorder=1) - ax.set_xscale("log") - ax.set_xlim(xlim) - ax.set_ylim(ylim) - ax.set_xlabel(r"$\theta$ (arcmin)") - ax.set_title(label) - if show_legend: - ax.legend(loc="upper left") - - axes[0].set_ylabel(r"$\theta\xi \times 10^4$") - - fig.tight_layout() - output_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def _get_stats_row(chain, label: str, color: str) -> list[str | float]: - margestats = chain.getMargeStats() - try: - s8_stats = margestats.parWithName("S_8") - except Exception: - s8_stats = margestats.parWithName("s_8_input") - sigma8_stats = margestats.parWithName("SIGMA_8") - omegam_stats = margestats.parWithName("OMEGA_M") - return [ - label, - color, - s8_stats.mean, - s8_stats.mean - s8_stats.limits[0].lower, - s8_stats.limits[0].upper - s8_stats.mean, - sigma8_stats.mean, - sigma8_stats.mean - sigma8_stats.limits[0].lower, - sigma8_stats.limits[0].upper - sigma8_stats.mean, - omegam_stats.mean, - omegam_stats.mean - omegam_stats.limits[0].lower, - omegam_stats.limits[0].upper - omegam_stats.mean, - ] - - -def get_sigma_tension(mean1, low1, high1, mean2, low2, high2): - sigma1 = 0.5 * (high1 + low1) - sigma2 = 0.5 * (high2 + low2) - delta_mean = np.abs(mean1 - mean2) - sigma_tension = delta_mean / np.sqrt(sigma1**2 + sigma2**2) - sign = 1 if mean1 > mean2 else -1 - return sigma_tension * sign - - -def plot_cell_ee_with_bestfit( - cosmosis_data_path: str, - bestfit_specs: list[tuple[str, str, dict]], - output_folder: str, - output_path: Path, - ell_min: float = 10.0, - ell_max: float = 2048.0, - label_data: str = "Fiducial data", -) -> None: - """Extracted from get_chi2_cell.ipynb plot_best_fit() (cell 21-22). - - Parameters - ---------- - cosmosis_data_path : str - Path to CosmoSIS FITS file with CELL_EE and COVMAT HDUs. - bestfit_specs : list of (label, root, line_args_dict) - Each entry is (legend label, chain root name, dict of plot kwargs). - output_folder : str - Base path to best_fit directories (e.g. /n09data/guerrini/output_chains/). - output_path : Path - Where to save the figure. - """ - data = fits.getdata(cosmosis_data_path, "CELL_EE") - cov_mat = fits.getdata(cosmosis_data_path, "COVMAT") - - fig, ax = plt.subplots(1, 1, figsize=(8, 5)) - - ell = data["ANG"] - cell = data["VALUE"] - ax.errorbar( - ell, - ell * cell, - yerr=ell * np.sqrt(np.diag(cov_mat)), - fmt="o", - label=label_data, - color="black", - capsize=2, - ) - - for label, root, line_kw in bestfit_specs: - ell_th = np.loadtxt(f"{output_folder}/best_fit/{root}/shear_cl/ell.txt") - shear_cl = np.loadtxt(f"{output_folder}/best_fit/{root}/shear_cl/bin_1_1.txt") - mask = (ell_th > ell_min) & (ell_th < ell_max) - ax.plot(ell_th[mask], ell_th[mask] * shear_cl[mask], label=label, **line_kw) - - ax.axvline(x=1800, color="black", linestyle="--", alpha=0.5) - ax.axvline(x=2048, color="black", linestyle="--", alpha=1.0) - ax.axvline(x=500, color="black", linestyle="--", alpha=0.3) - - ax.text( - 1740, - 0.90, - r"$k_\mathrm{max} = 3 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, - ) - ax.text( - 1978, - 0.90, - r"$k_\mathrm{max} = 5 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, - ) - ax.text( - 470, - 0.90, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, - ) - - ax.set_ylabel(r"$\ell C_\ell$", fontsize=16) - ax.set_xlabel(r"$\ell$", fontsize=16) - ax.set_xlim(ell.min() - 10, ell.max() + 100) - ax.set_xscale("squareroot") - ax.set_xticks(np.array([100, 400, 900, 1600])) - ax.minorticks_on() - ax.tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax.xaxis.set_ticks(minor_ticks, minor=True) - ax.tick_params(axis="both", which="major", labelsize=14) - ax.tick_params(axis="both", which="minor", labelsize=10) - ax.yaxis.get_offset_text().set_fontsize(14) - - plt.legend(loc="lower center", bbox_to_anchor=(0.685, 0.70), fontsize=12) - - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, bbox_inches="tight") - plt.close(fig) - - -def plot_s8_whisker( - chain_specs: list[ChainSpec], - output_path: Path, - reference_labels: list[str] | None = None, - reference_colors: list[str] | None = None, - reference_label: str | None = None, -) -> None: - """Extracted from 2025_10_28_plot_whisker.ipynb (cells 5-11). - - Supports multiple reference bands: pass reference_labels and - reference_colors as parallel lists. Each reference gets its own - shaded band in the corresponding color. For backwards compatibility, - a single reference_label still works. - """ - if reference_labels is None and reference_label is not None: - reference_labels = [reference_label] - reference_colors = reference_colors or [None] - - g, chains = load_getdist_chains( - chain_specs, - width_inch=30, - axes_fontsize=60, - axes_labelsize=60, - legend_fontsize=60, - ) - plt.close(g.fig) - - labels = [spec.label for spec in chain_specs] - colours = [spec.color for spec in chain_specs] - alphas = [spec.alpha for spec in chain_specs] - - param_values = np.array( - [ - [ - "# Expt", - "Colour", - "S8_Mean", - "S8_low", - "S8_high", - "sigma_8_Mean", - "sigma_8_low", - "sigma_8_high", - "Omega_m_Mean", - "Omega_m_low", - "Omega_m_high", - ] - ], - dtype=object, - ) - - escaped_labels = np.char.replace(np.array(labels), "\\", "\\\\") - for i, chain in enumerate(chains): - row = _get_stats_row(chain, escaped_labels[i], colours[i]) - param_values = np.vstack((param_values, row)) - - expt = np.char.replace(param_values[1:, 0].astype(str), "\\\\", "\\") - colours_arr = param_values[1:, 1].astype(str) - s8_mean = param_values[1:, 2].astype(np.float64) - s8_low = param_values[1:, 3].astype(np.float64) - s8_high = param_values[1:, 4].astype(np.float64) - sigma8_mean = param_values[1:, 5].astype(np.float64) - sigma8_low = param_values[1:, 6].astype(np.float64) - sigma8_high = param_values[1:, 7].astype(np.float64) - omegam_mean = param_values[1:, 8].astype(np.float64) - omegam_low = param_values[1:, 9].astype(np.float64) - omegam_high = param_values[1:, 10].astype(np.float64) - - ref_indices = [] - for rl in reference_labels or []: - matches = np.where(expt == rl)[0] - if len(matches): - ref_indices.append(matches[0]) - ref_label_set = set(reference_labels or []) - - n_rows = len(expt) - fig_height = max(6, 0.5 * n_rows + 1) - fig = plt.figure(figsize=(10, fig_height)) - gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5]) - ax1 = fig.add_subplot(gs[0]) - ax2 = fig.add_subplot(gs[1], sharey=ax1) - ax3 = fig.add_subplot(gs[2], sharey=ax1) - - axs = [ax1, ax2, ax3] - - params = [ - (s8_mean, s8_low, s8_high, r"$S_8$"), - (sigma8_mean, sigma8_low, sigma8_high, r"$\sigma_8$"), - (omegam_mean, omegam_low, omegam_high, r"$\Omega_{\rm m}$"), - ] - - row_spacing = 0.1 - y = np.arange(len(expt)) - - for ax, param in zip(axs, params): - means, lows, highs, label = param - for i, mean, low, high, color, alpha in zip( - y, means, lows, highs, colours_arr, alphas - ): - ax.errorbar( - mean, - 0.05 + i * row_spacing, - xerr=np.array([low, high])[:, None], - fmt="o", - color=color, - ecolor=color, - elinewidth=2, - capsize=3, - alpha=alpha, - ) - ax.set_xlabel(label, fontsize=14) - - for ri, ref_idx in enumerate(ref_indices): - band_color = ( - reference_colors[ri] - if reference_colors and ri < len(reference_colors) - else colours_arr[ref_idx] - ) or colours_arr[ref_idx] - ax.axvspan( - means[ref_idx] - lows[ref_idx], - means[ref_idx] + highs[ref_idx], - color=band_color, - alpha=0.15, - zorder=0, - ) - - ax.grid(False) - ax.tick_params(axis="y", left=False, labelleft=False) - if label == r"$S_8$": - ax.set_xlim(0.25, 1.05) - elif label == r"$\sigma_8$": - ax.set_xlim(0.5, 1.2) - elif label == r"$\Omega_{\rm m}$": - ax.set_xlim(0.1, 0.5) - - axs[0].set_yticks(0.05 + y * row_spacing) - axs[0].set_yticklabels([]) - for label, color, alpha in zip(expt, colours_arr, alphas): - idx = np.where(expt == label)[0][0] - yloc = 0.05 + row_spacing * idx - axs[0].text( - 0.26, - yloc, - label, - fontsize=12, - ha="left", - va="center", - color=color, - alpha=alpha, - ) - if label not in ref_label_set and ref_indices: - ri0 = ref_indices[0] - s8_tension = get_sigma_tension( - s8_mean[idx], - s8_low[idx], - s8_high[idx], - s8_mean[ri0], - s8_low[ri0], - s8_high[ri0], - ) - sign_str = "+" if s8_tension > 0 else "-" - axs[0].text( - 1.045, - yloc, - rf"${sign_str}{np.abs(s8_tension):.2f}" + r"\, \sigma$", - fontsize=10, - ha="right", - va="center", - color=color, - alpha=alpha, - ) - - plt.gca().invert_yaxis() - plt.tight_layout() - - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -# ── Snakemake entry ────────────────────────────────────────────────────────── - - -def _config_from_snakemake(smk) -> CeremonyConfig: - """Build config from snakemake.input / snakemake.output / snakemake.params.""" - chain_root_dir = Path(smk.params.chain_root_dir) - - return CeremonyConfig( - blind=smk.params.blind, - chain_version=smk.params.chain_version, - chain_prefix=smk.params.chain_prefix, - chain_root_dir=chain_root_dir, - external_root_dir=chain_root_dir / "ext_data", - results_dir=Path(smk.params.results_dir), - evidence_dir=Path(smk.output.evidence).parent, - xi_data_path=Path(smk.input.xi_data), - pure_eb_path=Path(smk.input.pure_eb), - pseudo_cl_path=Path(smk.input.pseudo_cl), - pseudo_cl_cov_path=Path(smk.input.pseudo_cl_cov), - cosmosis_cell_fits=Path(smk.input.cosmosis_cell_fits), - bestfit_dir=Path(smk.input.bestfit_dir).parent.parent, - bestfit_root_fid_cell=smk.params.bestfit_root_fid_cell, - bestfit_root_halofit_cell=smk.params.bestfit_root_halofit_cell, - bestfit_root_config=smk.params.bestfit_root_config, - ) - - -# ── CLI entry ──────────────────────────────────────────────────────────────── - -_CHAIN_ROOT_DIR = Path("/n09data/guerrini/output_chains") -_COSMOSIS_DATA_DIR = Path("/home/guerrini/sp_validation/cosmo_inference/data") -_DEFAULT_CHAIN_VERSION = "v1.4.6" - - -def _require_path(path: Path, label: str) -> Path: - if path.exists(): - return path - raise FileNotFoundError(f"{label}: {path}") - - -def _require_bestfit_root(chain_root_dir: Path, root: str) -> str: - if (chain_root_dir / "best_fit" / root / "shear_cl" / "ell.txt").exists(): - return root - raise FileNotFoundError(f"No best-fit shear_cl for {root}") - - -def _config_from_cli() -> CeremonyConfig: - """Build config from command-line arguments + path resolution. - - Uses the exact data vectors from inference (Lisa's xi_pm, Sasha's pseudo-Cl - and CosmoSIS FITS) — the same files the chains were fit to. - """ - _PROJECT_ROOT = _SCRIPT_DIR.parent.parent - - parser = argparse.ArgumentParser( - description="Run the UNIONS unblinding ceremony plot sequence." - ) - parser.add_argument("blind", choices=["A", "B", "C"], help="Revealed blind letter") - parser.add_argument( - "--chain-version", - default=_DEFAULT_CHAIN_VERSION, - help="Chain version (default: %(default)s)", - ) - parser.add_argument( - "--output-dir", - type=Path, - default=None, - help="Output directory for results (default: /results/unblinding)", - ) - args = parser.parse_args() - - blind = args.blind - chain_version = args.chain_version - chain_prefix = f"SP_{chain_version}_leak_corr" - output_dir = args.output_dir or (_PROJECT_ROOT / "results" / "unblinding") - - xi_data_path = _require_path( - _COSMOSIS_DATA_DIR - / f"{chain_prefix}_{blind}" - / f"cosmosis_{chain_prefix}_{blind}.fits", - f"CosmoSIS xi FITS for blind {blind}", - ) - - pseudo_cl_path = _require_path( - Path( - f"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_{chain_prefix}.fits" - ), - f"pseudo-Cl for {chain_prefix} (Guerrini)", - ) - pseudo_cl_cov_path = _require_path( - Path( - f"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_cov_{chain_prefix}.fits" - ), - f"pseudo-Cl covariance for {chain_prefix} (Guerrini)", - ) - - cosmosis_cell_fits = _require_path( - _COSMOSIS_DATA_DIR - / f"{chain_prefix}_{blind}_fid" - / f"cosmosis_{chain_prefix}_{blind}_fid_cell.fits", - f"CosmoSIS C_ell FITS for blind {blind}", - ) - - bestfit_root_config = f"{chain_prefix}_{blind}_10_80" - bestfit_dir = _require_path( - _CHAIN_ROOT_DIR / "best_fit" / bestfit_root_config, - f"Best-fit directory for blind {blind}", - ) - - return CeremonyConfig( - blind=blind, - chain_version=chain_version, - chain_prefix=chain_prefix, - chain_root_dir=_CHAIN_ROOT_DIR, - external_root_dir=_CHAIN_ROOT_DIR / "ext_data", - results_dir=output_dir, - evidence_dir=output_dir / "claims" / "unblinding_ceremony", - xi_data_path=xi_data_path, - pure_eb_path=_require_path( - _PROJECT_ROOT - / "results" - / "paper_plots" - / "intermediate" - / f"{chain_prefix}_{blind}_pure_eb_semianalytic.npz", - f"Pure E/B file for blind {blind}", - ), - pseudo_cl_path=pseudo_cl_path, - pseudo_cl_cov_path=pseudo_cl_cov_path, - cosmosis_cell_fits=cosmosis_cell_fits, - bestfit_dir=bestfit_dir, - bestfit_root_fid_cell=_require_bestfit_root( - _CHAIN_ROOT_DIR, f"{chain_prefix}_{blind}_fid_cell" - ), - bestfit_root_halofit_cell=_require_bestfit_root( - _CHAIN_ROOT_DIR, f"{chain_prefix}_{blind}_halofit_cell" - ), - bestfit_root_config=bestfit_root_config, - ) - - -# ── Main ceremony ──────────────────────────────────────────────────────────── - - -def run_ceremony(cfg: CeremonyConfig) -> None: - blind = cfg.blind - cfg.results_dir.mkdir(parents=True, exist_ok=True) - - reveal_harmonic_root = f"{cfg.chain_prefix}_{blind}_lmin=300_lmax=1600_cell" - reveal_config_root = f"{cfg.chain_prefix}_{blind}_10_80" - - # ── Act 1 — The Data (naked, then with fits) ────────────────────── - - # 01: xi+/- data + B-modes, no theory - plot_xipm_bestfit_with_bmodes( - xi_data_path=cfg.xi_data_path, - pure_eb_data_path=cfg.pure_eb_path, - bestfit_dir=None, - output_path=_save_path(cfg, 1, "xi_pm_data"), - ) - - # 02: C_ell^EE data, no theory - plot_cell_ee_data_vector( - str(cfg.pseudo_cl_path), - str(cfg.pseudo_cl_cov_path), - _save_path(cfg, 2, "cell_ee_data"), - ) - - # 03: xi+/- with config-space best-fit (Paper IV Fig 1) - plot_xipm_bestfit_with_bmodes( - xi_data_path=cfg.xi_data_path, - pure_eb_data_path=cfg.pure_eb_path, - bestfit_dir=cfg.bestfit_dir, - output_path=_save_path(cfg, 3, f"xi_bestfit_blind_{blind}"), - ) - - # 04: C_ell^EE with harmonic + config best-fit (Paper V Fig 2) - plot_cell_ee_with_bestfit( - cosmosis_data_path=str(cfg.cosmosis_cell_fits), - bestfit_specs=[ - ( - rf"UNIONS $C_\ell$, Blind {blind}", - cfg.bestfit_root_fid_cell, - {"color": "royalblue", "linestyle": "-"}, - ), - ( - r"UNIONS $C_\ell$, Halofit", - cfg.bestfit_root_halofit_cell, - {"color": "royalblue", "linestyle": "--"}, - ), - ( - r"UNIONS $\xi_\pm(\vartheta)$ (Goh et al., 2026)", - cfg.bestfit_root_config, - {"color": "orange", "linestyle": "-"}, - ), - ], - output_folder=str(cfg.chain_root_dir), - output_path=_save_path(cfg, 4, f"cell_ee_bestfit_blind_{blind}"), - ) - - # ── Act 2 — The Reveal ──────────────────────────────────────────── - - # 05: Consistency — Omega_m-S8, harmonic vs config vs Planck - plot_triangle( - [ - ChainSpec( - root=reveal_harmonic_root, - label=rf"UNIONS $C_\ell$, Blind {blind}", - color="royalblue", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root=reveal_config_root, - label=rf"UNIONS $\xi_\pm(\vartheta)$, Blind {blind}", - color="orange", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root="Planck18", - label=r"\textit{Planck} 2018", - color="violet", - base_dir=cfg.external_root_dir, - ), - ], - ["OMEGA_M", "S_8"], - _save_path(cfg, 5, f"consistency_blind_{blind}"), - width_inch=12, - axes_fontsize=24, - axes_labelsize=28, - ) - - # 06: 4-param triangle — revealed blind, harmonic + config overlaid - plot_triangle( - [ - ChainSpec( - root=reveal_harmonic_root, - label=rf"UNIONS $C_\ell$, Blind {blind}", - color="royalblue", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root=reveal_config_root, - label=rf"UNIONS $\xi_\pm(\vartheta)$, Blind {blind}", - color="orange", - base_dir=cfg.chain_root_dir, - ), - ], - COSMO_PARAMS, - _save_path(cfg, 6, f"triangle_cosmo_blind_{blind}"), - width_inch=20, - ) - - # 07: Full-param triangle — revealed blind, harmonic + config overlaid - plot_triangle( - [ - ChainSpec( - root=reveal_harmonic_root, - label=rf"UNIONS $C_\ell$, Blind {blind}", - color="royalblue", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root=reveal_config_root, - label=rf"UNIONS $\xi_\pm(\vartheta)$, Blind {blind}", - color="orange", - base_dir=cfg.chain_root_dir, - ), - ], - FULL_PARAMS, - _save_path(cfg, 7, f"triangle_full_blind_{blind}"), - width_inch=30, - axes_fontsize=26, - axes_labelsize=28, - ) - - # ── Act 3 — In Context ──────────────────────────────────────────── - - # 08: S8 whisker — all 6 UNIONS blinds (3 harmonic + 3 config), - # revealed blind highlighted, rest muted. Plus external surveys. - whisker_specs = [] - - for b in ("A", "B", "C"): - is_revealed = b == blind - alpha = 1.0 if is_revealed else MUTED_ALPHA - whisker_specs.append( - ChainSpec( - root=f"{cfg.chain_prefix}_{b}_lmin=300_lmax=1600_cell", - label=rf"UNIONS $C_\ell$, Blind {b}", - color="royalblue", - base_dir=cfg.chain_root_dir, - alpha=alpha, - ) - ) - whisker_specs.append( - ChainSpec( - root=f"{cfg.chain_prefix}_{b}_10_80", - label=rf"UNIONS $\xi_\pm$, Blind {b}", - color="orange", - base_dir=cfg.chain_root_dir, - alpha=alpha, - ) - ) - - whisker_specs.extend( - [ - ChainSpec( - root="Planck18", - label=r"\textit{Planck} 2018", - color="black", - base_dir=cfg.external_root_dir, - ), - ChainSpec( - root="DES_Y3", - label=r"DES Y3 $\xi_\pm$", - color="black", - base_dir=cfg.external_root_dir, - ), - ChainSpec( - root="KiDS-1000", - label=r"KiDS-1000 $\xi_\pm$", - color="black", - base_dir=cfg.external_root_dir, - ), - ChainSpec( - root="HSC_Y3", - label=r"HSC Y3 $\xi_\pm$", - color="black", - base_dir=cfg.external_root_dir, - ), - ] - ) - - plot_s8_whisker( - whisker_specs, - reference_labels=[ - rf"UNIONS $C_\ell$, Blind {blind}", - rf"UNIONS $\xi_\pm$, Blind {blind}", - ], - reference_colors=["royalblue", "orange"], - output_path=_save_path(cfg, 8, f"s8_whisker_blind_{blind}"), - ) - - # ── Write evidence ─────────────────────────────────────────────── - - produced_figures = [ - _save_path(cfg, 1, "xi_pm_data"), - _save_path(cfg, 2, "cell_ee_data"), - _save_path(cfg, 3, f"xi_bestfit_blind_{blind}"), - _save_path(cfg, 4, f"cell_ee_bestfit_blind_{blind}"), - _save_path(cfg, 5, f"consistency_blind_{blind}"), - _save_path(cfg, 6, f"triangle_cosmo_blind_{blind}"), - _save_path(cfg, 7, f"triangle_full_blind_{blind}"), - _save_path(cfg, 8, f"s8_whisker_blind_{blind}"), - ] - output_dict = {} - for fig_path in produced_figures: - output_dict[fig_path.stem] = fig_path.name - - cfg.evidence_dir.mkdir(parents=True, exist_ok=True) - - evidence = { - "id": "unblinding_ceremony", - "spec_id": "unblinding_ceremony", - "generated": datetime.now(timezone.utc).isoformat(timespec="seconds"), - "input": { - "xi_data": str(cfg.xi_data_path), - "pure_eb_data": str(cfg.pure_eb_path), - "pseudo_cl": str(cfg.pseudo_cl_path), - "pseudo_cl_cov": str(cfg.pseudo_cl_cov_path), - "cosmosis_cell_fits": str(cfg.cosmosis_cell_fits), - "harmonic_chains": str( - cfg.chain_root_dir - / f"{cfg.chain_prefix}_{{A,B,C}}_lmin=300_lmax=1600_cell" - ), - "config_chains": str( - cfg.chain_root_dir / f"{cfg.chain_prefix}_{{A,B,C}}_10_80" - ), - "external_chains": str( - cfg.external_root_dir / "{Planck18,DES_Y3,KiDS-1000,HSC_Y3}" - ), - }, - "output": output_dict, - "params": { - "chain_version": cfg.chain_version, - "blind": blind, - "scale_cut_arcmin": "12-83", - "ell_range": "300-1600", - "n_figures": len(output_dict), - }, - "evidence": { - "ceremony_date": "2026-02-27", - "script": "workflow/scripts/unblinding_ceremony.py", - }, - } - - evidence_path = cfg.evidence_dir / "evidence.json" - evidence_path.write_text(json.dumps(evidence, indent=2) + "\n") - - for fig_path in produced_figures: - shutil.copy2(fig_path, cfg.evidence_dir / fig_path.name) - - print(f"Saved ceremony figures to {cfg.results_dir.resolve()}") - print(f"Wrote evidence to {evidence_path}") - - -# ── Entry point dispatch ───────────────────────────────────────────────────── - -try: - snakemake # injected by snakemake's script: directive -except NameError: - snakemake = None - -if snakemake is not None: - run_ceremony(_config_from_snakemake(snakemake)) -elif __name__ == "__main__": - run_ceremony(_config_from_cli()) diff --git a/papers/cosmo_val/Snakefile b/papers/cosmo_val/Snakefile index 2585bb7a..716a3ac6 100644 --- a/papers/cosmo_val/Snakefile +++ b/papers/cosmo_val/Snakefile @@ -9,8 +9,6 @@ configfile: "config/config.yaml" configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" -container: "/n17data/cdaley/containers/containers" - envvars: "PYTHONUNBUFFERED", @@ -32,6 +30,9 @@ import common common.configure(config) from common import * +# The one image every rule runs in; see common.resolve_container. +container: common.resolve_container(config.get("container")) + # Wildcard constraints — centralized in common.py, not in individual rule files wildcard_constraints: **WILDCARD_CONSTRAINTS diff --git a/pyproject.toml b/pyproject.toml index c2a9aa2c..071945b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,13 @@ dependencies = [ [project.urls] Homepage = "https://github.com/CosmoStat/sp_validation" +[project.scripts] +# Host-side management of your own container image (sp_validation.container). +# Stdlib-only by design -- it has to run on the host, outside the container -- +# so it also works straight from a checkout with nothing installed: +# `python3 src/sp_validation/container.py status`. +spv-container = "sp_validation.container:main" + [tool.uv] # The reproducibility target is the Linux container; scope the lock to Linux # (mirrors shapepipe) so `uv lock` resolves the linux-centric stack (pymaster, @@ -132,8 +139,8 @@ docs = [ "sphinxawesome-theme>=5.3,!=6.0.3" ] # GLASS mock generation (sp_validation.glass_mock). Kept optional: the core -# library and its import guard resolve without GLASS, and the production -# container does not yet ship it. Pinned set (verified 2026-06-13, fiber +# library and its import guard resolve without GLASS (the production container +# ships it via the ``glass`` extra). Pinned set (verified 2026-06-13, fiber # glass-cosmology-api-pin): glass 2025.1 is the unique version with the flat API # the map path uses AND the legacy ``cosmo.dc``/``xm``/``ef`` interface that # ``cosmology`` 2022.10.9 (its newest release) provides — newer glass calls @@ -153,8 +160,18 @@ glass = [ workflow = [ "snakemake", # run_2pcf_highres.py drives the MPI convergence run; the container ships - # OpenMPI (/opt/ompi) so mpi4py builds against it. + # OpenMPI under /opt/ompi so mpi4py builds against it. "mpi4py", + # CosmoSIS drives the cosmo_inference sampling step. Sdist-only, so this is a + # source build (~2 min) against the base image's gfortran/GSL/cfitsio/OpenMPI — + # no extra apt packages needed. Floor at 3.25 simply to stay current; there is + # no known-good older version to pin back to. Note the shared-memory pool is + # still broken upstream at 3.25.2 (bcast/gather/allreduce in + # runtime/process_pool.py return an unassigned `self.data`), so chains must run + # under MPI -- which is why the Dockerfile sets MPIFC before syncing this extra. + "cosmosis>=3.25", + # CSL's project_2d.py imports fastpt (3.x API; 4.0 restructured it). + "fast-pt>=3.2,<4", # NOTE: workflow/scripts/cv_*.py also import `cv_runner`, which is not # published or resolvable (no public repo found) — left undeclared pending # its source. Same for `unions_wl` (scripts/check_footprint.py). diff --git a/src/sp_validation/container.py b/src/sp_validation/container.py new file mode 100755 index 00000000..7aa5f2ed --- /dev/null +++ b/src/sp_validation/container.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""Manage this user's local copy of the sp_validation container image. + +Everyone runs their own image: the canonical paths are under your own cache, you +refresh them when you want to, and nobody else's refresh moves the ground under +a running job. + +There are two layers, and you only need the second when you want it: + +* the **SIF** (``~/.cache/sp_validation/sp_validation.sif``) -- a pristine, + read-only copy of the published image. This is the default and the normal case. +* an optional **sandbox** (``~/.cache/sp_validation/sandbox/``) -- the same image + unpacked into a writable directory, so ``pip install`` inside it sticks. This + is the escape hatch for exploratory work that needs a package the image does + not carry yet, and it is opt-in: nothing builds one for you. + +Subcommands, exposed as the ``spv-container`` console script:: + + spv-container pull # fetch the tag to the canonical path + spv-container status # what is here, and how current is it + spv-container sandbox # unpack the SIF into a writable dir + spv-container exec # run something inside it + spv-container exec --writable # ... with writes that persist + +Everything resolves the same image in the same order -- **sandbox if it exists, +else the SIF, else the registry tag** -- and that includes the Snakemake +workflow, so a package you installed into your sandbox is there for your +workflow jobs too. + +This module is deliberately **stdlib-only** (``argparse``/``subprocess``/ +``pathlib``). It runs on the *host*, outside the container, where the science +stack is not installed -- so it must import without it. That also means it works +straight from a checkout with no install at all:: + + python3 src/sp_validation/container.py pull +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# The image every entry point names, written down here once. CI pushes one tag +# per branch, sanitized, so ``:develop`` tracks the integration branch. +CONTAINER_URI = "docker://ghcr.io/cosmostat/sp_validation:develop" + +CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "sp_validation" + +# Where this user's image lives. Per-user by construction: one file, one owner, +# no coordination. Override with ``SPV_CONTAINER`` (an absolute path). +DEFAULT_SIF = CACHE_DIR / "sp_validation.sif" + +# The optional writable unpacking of that image. Override with ``SPV_SANDBOX``. +DEFAULT_SANDBOX = CACHE_DIR / "sandbox" + +# Bind mounts for interactive `exec`. candide's disks; override wholesale with +# ``SPV_APPTAINER_BINDS`` or per-call with ``--bind``. +DEFAULT_BINDS = "/home,/scratch,/automnt,/n17data,/n23data1,/n09data" + + +def local_sif(): + """Return this user's canonical image path (may not exist yet).""" + override = os.environ.get("SPV_CONTAINER") + path = Path(override) if override else DEFAULT_SIF + return path.expanduser() + + +def local_sandbox(): + """Return this user's writable sandbox directory (may not exist).""" + override = os.environ.get("SPV_SANDBOX") + path = Path(override) if override else DEFAULT_SANDBOX + return path.expanduser() + + +def resolve_image(): + """Return ``(path_or_uri, kind)`` for the image everything should run. + + The one resolution order, shared by the CLI and the workflow: the writable + sandbox if it exists, else the pristine SIF if it exists, else the registry + tag for Snakemake to pull. ``kind`` is ``"sandbox"``, ``"sif"`` or ``"tag"``. + """ + sandbox = local_sandbox() + if sandbox.is_dir(): + return str(sandbox), "sandbox" + sif = local_sif() + if sif.exists(): + return str(sif), "sif" + return CONTAINER_URI, "tag" + + +def image_labels(sif): + """Return the image's OCI labels as a dict, or ``{}`` if unreadable. + + Never raises: a missing file, a missing ``apptainer``, or a corrupt image + all mean "we don't know", which every caller here treats as non-fatal. + """ + sif = Path(sif) + if not sif.exists() or shutil.which("apptainer") is None: + return {} + try: + out = subprocess.run( + ["apptainer", "inspect", "--labels", str(sif)], + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return {} + if out.returncode != 0: + return {} + labels = {} + for line in out.stdout.splitlines(): + key, sep, value = line.partition(":") + if sep: + labels[key.strip()] = value.strip() + return labels + + +def image_revision(sif): + """Return the sp_validation commit the image was built from, or ``None``.""" + return image_labels(sif).get("org.opencontainers.image.revision") + + +def _require_apptainer(): + """Exit unless ``apptainer`` is on PATH.""" + if shutil.which("apptainer") is None: + sys.exit("apptainer is not on PATH") + + +def _git(*args, cwd=None): + """Run a git command, returning stripped stdout or ``None`` on any failure.""" + try: + out = subprocess.run( + ["git", *args], capture_output=True, text=True, cwd=cwd, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + +def compare_revision(revision, repo=None): + """Place an image revision relative to a checkout's HEAD. + + Returns one of ``"in-sync"``, ``"behind"`` (the image predates HEAD), + ``"ahead"`` (HEAD predates the image), ``"diverged"``, or ``"unknown"`` + (no revision label, no git, or a commit this clone has never fetched). + """ + if not revision: + return "unknown" + repo = repo or Path(__file__).resolve().parents[2] + head = _git("rev-parse", "HEAD", cwd=repo) + if head is None: + return "unknown" + if head == revision: + return "in-sync" + if _git("cat-file", "-e", f"{revision}^{{commit}}", cwd=repo) is None: + return "unknown" + if _git("merge-base", "--is-ancestor", revision, head, cwd=repo) is not None: + return "behind" + if _git("merge-base", "--is-ancestor", head, revision, cwd=repo) is not None: + return "ahead" + return "diverged" + + +def cmd_pull(args): + """Pull ``--tag`` to the canonical path, atomically.""" + _require_apptainer() + sif = local_sif() + sif.parent.mkdir(parents=True, exist_ok=True) + # Pull to a sibling temp name and rename: an atomic rename within one + # directory, so a job gets either the whole old image or the whole new one. + # Pulling in place would leave the file half-written for the ~15 minutes the + # pull takes. Jobs already running hold the old inode open and finish on it. + tmp = sif.with_name(sif.name + f".pull.{os.getpid()}") + print(f"pulling {args.tag}\n -> {sif}") + try: + subprocess.run( + ["apptainer", "pull", "--force", "--name", str(tmp), args.tag], check=True + ) + os.replace(tmp, sif) + except subprocess.CalledProcessError as exc: + tmp.unlink(missing_ok=True) + sys.exit(f"pull failed ({exc.returncode})") + except KeyboardInterrupt: + tmp.unlink(missing_ok=True) + raise + labels = image_labels(sif) + print(f"revision: {labels.get('org.opencontainers.image.revision', 'unknown')}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + return 0 + + +def cmd_sandbox(args): + """Unpack the image into a writable directory -- the opt-in escape hatch.""" + _require_apptainer() + sandbox = local_sandbox() + if sandbox.exists() and not args.force: + sys.exit( + f"sandbox already exists at {sandbox}\n" + "pass --force to discard it and rebuild from a clean image" + ) + source = args.source or ( + str(local_sif()) if local_sif().exists() else CONTAINER_URI + ) + sandbox.parent.mkdir(parents=True, exist_ok=True) + print(f"building sandbox from {source}\n -> {sandbox}") + # Build beside the target and swap it in, as `pull` does -- and for a sharper + # reason here. A half-written .sif fails loudly, but a half-unpacked sandbox + # *directory* is still a directory, so resolve_image() would elect it as the + # live image and every job would silently run a broken tree. + # + # Building first also means a `--force` rebuild that fails (a typo in + # --source, a network blip) leaves the sandbox you already had untouched, + # rather than deleting a working environment on the way to not replacing it. + # + # `--fix-perms` so the tree can be deleted again later (apptainer warns about + # exactly this otherwise). No `--fakeroot`: an unprivileged build from an + # existing image works through user namespaces, which is what candide has. + staging = sandbox.with_name(f"{sandbox.name}.build.{os.getpid()}") + shutil.rmtree(staging, ignore_errors=True) + try: + subprocess.run( + ["apptainer", "build", "--sandbox", "--fix-perms", str(staging), source], + check=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"sandbox build failed ({exc.returncode}); {sandbox} is unchanged") + except (KeyboardInterrupt, OSError): + shutil.rmtree(staging, ignore_errors=True) + raise + + if sandbox.exists(): + print(f"replacing {sandbox}") + shutil.rmtree(sandbox, ignore_errors=True) + if sandbox.exists(): + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"could not remove {sandbox}; remove it by hand and retry") + os.replace(staging, sandbox) + print( + "\nthis sandbox now takes precedence over the SIF everywhere, including " + "workflow jobs.\ninstall into it with: spv-container exec --writable pip " + "install \nreset to a clean image with: spv-container pull && " + "spv-container sandbox --force" + ) + return 0 + + +def cmd_status(args): + """Report which image layer is live, its revision, and how current it is.""" + sif = local_sif() + sandbox = local_sandbox() + active, kind = resolve_image() + + if sif.exists(): + print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") + else: + print(f"SIF: absent ({sif})") + if sandbox.is_dir(): + print(f"sandbox: {sandbox} (writable; may carry local modifications)") + else: + print("sandbox: none") + + if kind == "tag": + print(f"\nactive: {active} (registry tag -- nothing pulled locally)") + print("run: spv-container pull") + return 1 + + print(f"\nactive: {active} ({kind})") + labels = image_labels(active) + revision = labels.get("org.opencontainers.image.revision") + source = "" + if revision is None and kind == "sandbox" and sif.exists(): + # Some sandbox trees do not carry the original labels through. The SIF + # beside it is the best remaining evidence of what it was built from -- + # a guess, so it is labelled as one rather than printed as fact. + revision = image_revision(sif) + if revision: + source = " (inferred from the SIF beside it, not read from the sandbox)" + print(f"revision: {revision or 'unknown'}{source}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + if kind == "sandbox": + print( + " (the revision above is what the sandbox was built from; " + "anything\n installed into it since is not reflected in " + "any label)" + ) + verdict = compare_revision(revision) + explain = { + "in-sync": "matches this checkout's HEAD", + "behind": "older than this checkout's HEAD -- pull to refresh", + "ahead": "newer than this checkout's HEAD", + "diverged": "on a different branch from this checkout", + "unknown": "cannot compare (no label, or a commit this clone lacks)", + }[verdict] + print(f"checkout: {verdict} ({explain})") + return 0 + + +def cmd_exec(args): + """Run a command inside the image -- the one-off path for humans and agents.""" + _require_apptainer() + if not args.command: + sys.exit("nothing to run; pass a command after `exec`") + binds = args.bind or os.environ.get("SPV_APPTAINER_BINDS", DEFAULT_BINDS) + + if args.writable: + # A SIF is a read-only filesystem, so `--writable` against one fails + # obscurely; only a sandbox takes writes. + sandbox = local_sandbox() + if not sandbox.is_dir(): + sys.exit( + f"--writable needs a sandbox, and there is none at {sandbox}\n" + "build one with: spv-container sandbox" + ) + image, extra = str(sandbox), ["--writable"] + else: + image, kind = resolve_image() + if kind == "tag": + sys.exit(f"no local image; run: spv-container pull ({image})") + extra = [] + + cmd = [ + "apptainer", + "exec", + *extra, + "--cleanenv", + "--bind", + binds, + image, + *args.command, + ] + return subprocess.run(cmd).returncode + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="spv-container", description=__doc__.splitlines()[0] + ) + sub = parser.add_subparsers(dest="subcommand", required=True) + + p_pull = sub.add_parser("pull", help="fetch the image to the canonical path") + p_pull.add_argument( + "--tag", + default=CONTAINER_URI, + help=f"image to pull (default: {CONTAINER_URI})", + ) + p_pull.set_defaults(func=cmd_pull) + + p_status = sub.add_parser( + "status", help="report which image layer is live and how current it is" + ) + p_status.set_defaults(func=cmd_status) + + p_sandbox = sub.add_parser( + "sandbox", help="unpack the image into a writable directory (opt-in)" + ) + p_sandbox.add_argument( + "--source", + help="image to unpack (default: the local SIF, or the registry tag)", + ) + p_sandbox.add_argument( + "--force", + action="store_true", + help="discard an existing sandbox and rebuild from a clean image", + ) + p_sandbox.set_defaults(func=cmd_sandbox) + + p_exec = sub.add_parser("exec", help="run a command inside the local image") + p_exec.add_argument("--bind", help=f"bind mounts (default: {DEFAULT_BINDS})") + p_exec.add_argument( + "--writable", + action="store_true", + help="run against the sandbox so writes (e.g. pip install) persist", + ) + p_exec.add_argument("command", nargs=argparse.REMAINDER) + p_exec.set_defaults(func=cmd_exec) + + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sp_validation/tests/data/container_smoke/Snakefile b/src/sp_validation/tests/data/container_smoke/Snakefile new file mode 100644 index 00000000..75527690 --- /dev/null +++ b/src/sp_validation/tests/data/container_smoke/Snakefile @@ -0,0 +1,21 @@ +# Standalone workflow exercised by src/sp_validation/tests/test_container_smoke.py. +# +# The module-level `container:` below mirrors what every real workflow does +# (workflow/Snakefile) -- Snakemake has no way to take a default image from a +# profile. Everything else under test arrives from the driving profile +# (workflow/profiles/candide). See container_smoke.py for what the job checks. + + +# Literal rather than the package's CONTAINER_URI: this Snakefile is test data, +# deliberately standalone. test_container_smoke.py asserts the two agree, so +# drift fails the test rather than the run. +container: config.get("container", "docker://ghcr.io/cosmostat/sp_validation:develop") + + +rule container_smoke: + output: + "results/container_smoke.yaml", + resources: + runtime=5, + script: + "container_smoke.py" diff --git a/src/sp_validation/tests/data/container_smoke/container_smoke.py b/src/sp_validation/tests/data/container_smoke/container_smoke.py new file mode 100644 index 00000000..274cd85c --- /dev/null +++ b/src/sp_validation/tests/data/container_smoke/container_smoke.py @@ -0,0 +1,87 @@ +"""Rule container_smoke: exercise the containerized-SLURM path end to end. + +Cheap sanity check of the profile-driven container path -- same executor +(slurm), same software-deployment-method (apptainer), same apptainer-args +binds, same container image every real rule uses. Four things it proves, each +written to the output YAML: + + * the job really ran inside the image (``APPTAINER_CONTAINER``, set by + apptainer itself -- without it the rest could all pass on the bare host); + * the editable ``sp_validation`` install resolves on the container's + PYTHONPATH (import provenance: file + version, not just import success); + * the numeric stack works (numpy eigh on a small fixed matrix). + ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; + * which commit of this checkout is running (git rev-parse from inside the + container -- proves /home is bound and usable, not just readable). + +Driven by the co-located Snakefile; the assertions on the output YAML live in +src/sp_validation/tests/test_container_smoke.py (marked ``slow``, cluster only). +""" + +import os +import platform +import subprocess + +import numpy as np +import yaml + +# --- the job is actually inside the image --------------------------------- +container_info = { + "apptainer_container": os.environ.get("APPTAINER_CONTAINER", "unset"), +} + +# --- editable install resolves inside the container ------------------------ +import sp_validation # noqa: E402 + +sp_validation_info = { + "version": getattr(sp_validation, "__version__", "unknown"), + "file": sp_validation.__file__, +} + +# --- numeric stack + threading ----------------------------------------- +rng = np.random.default_rng(seed=42) +a = rng.standard_normal((8, 8)) +symmetric = a + a.T +eigenvalues = np.linalg.eigh(symmetric)[0] + +numeric_info = { + "numpy_version": np.__version__, + "eigenvalues": [float(v) for v in eigenvalues], + "omp_num_threads": os.environ.get("OMP_NUM_THREADS", "unset"), +} + +# --- provenance: what commit is actually running in the container --------- +# src/sp_validation/tests/data/container_smoke/ -> repo root, five levels up. +# (This is the checkout the Snakefile came from, which is what we want to +# report; the editable install may well resolve to a *different* checkout.) +repo_dir = os.path.abspath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 5)) +) +try: + commit = subprocess.run( + ["git", "-C", repo_dir, "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() +except (subprocess.CalledProcessError, FileNotFoundError) as exc: + commit = f"unavailable ({exc})" + +provenance = { + "repo_dir": repo_dir, + "commit": commit, + "hostname": platform.node(), + "python": platform.python_version(), +} + +with open(snakemake.output[0], "w") as f: + yaml.safe_dump( + { + "container": container_info, + "sp_validation": sp_validation_info, + "numeric": numeric_info, + "provenance": provenance, + }, + f, + sort_keys=False, + ) diff --git a/src/sp_validation/tests/test_container_smoke.py b/src/sp_validation/tests/test_container_smoke.py new file mode 100644 index 00000000..b9b07058 --- /dev/null +++ b/src/sp_validation/tests/test_container_smoke.py @@ -0,0 +1,126 @@ +"""Smoke test of the profile-driven containerized-SLURM path. + +Submits one real (tiny, 5-minute) SLURM job through the committed candide +profile. The executor, the apptainer deployment method and the bind mounts come +from that profile; the image is the module-level ``container:`` in the test +Snakefile, exactly as real workflows declare it. That contract is what's under +test, so this can only run on candide -- marked ``slow``, skipped elsewhere. + +The job writes a YAML report (see data/container_smoke/container_smoke.py); the +assertions below check what it reports. +""" + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import numpy as np +import pytest +import yaml + +requires_cluster = pytest.mark.skipif( + not Path("/n17data/cdaley/unions").exists() or shutil.which("sbatch") is None, + reason="needs candide: /n17data and a SLURM submit host", +) + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def _reference_eigenvalues() -> np.ndarray: + """The same deterministic computation the job runs inside the container.""" + rng = np.random.default_rng(seed=42) + a = rng.standard_normal((8, 8)) + return np.linalg.eigh(a + a.T)[0] + + +def test_smoke_snakefile_names_the_workflow_image(): + """The test Snakefile's literal image must track the package's CONTAINER_URI.""" + repo_root = _repo_root() + uri = re.search( + r'^CONTAINER_URI = "(.+)"$', + (repo_root / "src/sp_validation/container.py").read_text(), + re.MULTILINE, + ).group(1) + snakefile = ( + repo_root / "src/sp_validation/tests/data/container_smoke/Snakefile" + ).read_text() + assert f'"{uri}"' in snakefile, uri + + +@pytest.mark.slow +@requires_cluster +def test_container_smoke(): + repo_root = _repo_root() + workflow_dir = repo_root / "src/sp_validation/tests/data/container_smoke" + + # Not pytest's tmp_path: that lives in the login node's /tmp, which the + # compute node cannot see, so the job's output would "go missing". The + # workdir must be on a shared filesystem. + tmp_path = Path(tempfile.mkdtemp(prefix="container_smoke_", dir=Path.home())) + + env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + result = subprocess.run( + [ + "snakemake", + "--profile", + str(repo_root / "workflow/profiles/candide"), + "-s", + str(workflow_dir / "Snakefile"), + "--directory", + str(tmp_path), + "--jobs", + "1", + "container_smoke", + ], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=600, + check=False, + ) + assert result.returncode == 0, result.stdout + + report = yaml.safe_load((tmp_path / "results/container_smoke.yaml").read_text()) + + # The job ran inside the image, not on the bare host. Everything below would + # pass on the host too, so this is the assertion that makes them mean + # something: apptainer sets APPTAINER_CONTAINER in every process it starts. + assert report["container"]["apptainer_container"] != "unset", report["container"] + + # The install must resolve to an editable src/ checkout, not a site-packages + # copy. Note it need not be *this* checkout: the container's editable install + # points at the shared /n17data working tree, while the Snakefile under test + # is read from wherever the test runs. + module_file = Path(report["sp_validation"]["file"]) + assert module_file.parts[-3:] == ("src", "sp_validation", "__init__.py"), ( + module_file + ) + assert "site-packages" not in module_file.parts, module_file + + # The numeric stack agrees with the same computation run here. + np.testing.assert_allclose( + report["numeric"]["eigenvalues"], + _reference_eigenvalues(), + rtol=1e-10, + atol=1e-12, + ) + + # numeric.omp_num_threads is recorded but deliberately NOT asserted: the + # profile leaves OMP_NUM_THREADS unset by design, and rules that need it + # pinned set it themselves, so "unset" here is correct rather than a gap. + + # git worked inside the container, so /home is bound and usable. + assert re.fullmatch(r"[0-9a-f]{40}", report["provenance"]["commit"]), report[ + "provenance" + ] + + shutil.rmtree(tmp_path) # keep only on failure, for post-mortem diff --git a/src/sp_validation/tests/test_csl_modules.py b/src/sp_validation/tests/test_csl_modules.py new file mode 100644 index 00000000..941eccee --- /dev/null +++ b/src/sp_validation/tests/test_csl_modules.py @@ -0,0 +1,74 @@ +"""Every CSL module the pipeline templates reference must actually import. + +The image build only *compiles* CSL (``make -C shear``); pure-Python modules +are never loaded until CosmoSIS assembles a pipeline at runtime. That gap let +a scipy>=1.15 incompatibility (``scipy.special.lpn`` removed, imported by +``legendre.py`` via ``spec_tools`` — i.e. by every real-space likelihood) ship +in a "validated" image (#316). This test walks the ``file =`` entries of the +committed ini templates and imports each referenced ``.py`` module, so any +CSL-vs-environment drift on a module we actually use fails the in-image suite. + +Skips outside the image (no ``CSL_DIR`` / no cosmosis). +""" + +import importlib.util +import os +import re +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("cosmosis") + +CSL_DIR = os.environ.get("CSL_DIR") +if not CSL_DIR or not Path(CSL_DIR).is_dir(): + pytest.skip( + "CSL_DIR not set or missing (not in the image)", allow_module_level=True + ) + +TEMPLATES = ( + Path(__file__).parents[3] / "cosmo_inference" / "cosmosis_config" / "templates" +) + +_FILE_RE = re.compile(r"^file\s*=\s*%\((?:COSMOSIS_DIR|CSL_DIR)\)s/(.+)$") + + +def _template_modules(): + seen = set() + for ini in sorted(TEMPLATES.glob("*.ini")): + for line in ini.read_text().splitlines(): + m = _FILE_RE.match(line.strip()) + if m and m.group(1) not in seen: + seen.add(m.group(1)) + yield m.group(1) + + +MODULES = list(_template_modules()) + + +def test_templates_reference_csl_modules(): + """The parser found the template module list (guards against ini drift).""" + assert len(MODULES) >= 10 + + +@pytest.mark.parametrize("relpath", MODULES) +def test_csl_module_loads(relpath): + path = Path(CSL_DIR) / relpath + if path.suffix == ".so": + # Compiled in the image (and hard-checked by the Dockerfile's `test -f`); + # absent in a bare source checkout. + if not path.exists(): + pytest.skip(f"{relpath} not built in this CSL checkout") + return + assert path.exists(), f"template references missing CSL file: {relpath}" + # CosmoSIS modules import bare-named siblings (e.g. `import spec_tools`), + # so load with the module's own directory on sys.path, like CosmoSIS does. + sys.path.insert(0, str(path.parent)) + try: + name = "csl_probe_" + relpath.replace("/", "_").replace(".py", "") + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + finally: + sys.path.remove(str(path.parent)) diff --git a/uv.lock b/uv.lock index deedc1cf..4f635d14 100644 --- a/uv.lock +++ b/uv.lock @@ -561,6 +561,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2e/2c/c80db37593ad593e3f6c60705fbdf690b33f22c89edd5507d2731346cd87/cosmology-2022.10.9-py3-none-any.whl", hash = "sha256:3903658c2474177a1a1c75771ae14458d93200c516f8fc6c3f4d776f3287b288", size = 9341, upload-time = "2022-10-10T10:18:14.928Z" }, ] +[[package]] +name = "cosmosis" +version = "3.25.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dulwich" }, + { name = "dynesty" }, + { name = "emcee" }, + { name = "h5py" }, + { name = "matplotlib" }, + { name = "nautilus-sampler" }, + { name = "numpy" }, + { name = "py-bobyqa" }, + { name = "pybind11" }, + { name = "pyyaml" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "threadpoolctl" }, + { name = "zeus-mcmc" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/63/83898440e5486294cb17a7ddf01896af66dd153d7f3e77897ed120abd055/cosmosis-3.25.2.tar.gz", hash = "sha256:4a8333395b600a5e8339c637b947a982bac4c88a7b242e4b6de8274be03004a8", size = 416233, upload-time = "2026-03-09T12:01:08.269Z" } + [[package]] name = "coverage" version = "7.15.0" @@ -852,6 +874,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/91/428eca0cc8a1142b9952bef02c06bfe54526d730dd070de873424ab8c0bf/ducc0-0.41.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a3a71fd23661ddd6a7f6b44433e778d35a25b3f1492c9239217cdfedbe8178b1", size = 5613706, upload-time = "2026-03-26T18:55:24.971Z" }, ] +[[package]] +name = "dulwich" +version = "1.2.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/7b/d03c166f6098314824bdd2e661a32563c0ebe8c5e3344ed212d2ffc96a10/dulwich-1.2.14.tar.gz", hash = "sha256:ed8bfcfe1c7e187d1bc5e78f7c128a07d229d7b425433b04cbb89334678bd3b1", size = 1361695, upload-time = "2026-08-29T00:23:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/49/bca41ab16905963a1efb045f122e69f89e8ead682a3be17a02e0701aa099/dulwich-1.2.14-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:63ec5ccc659f5d8dcf6b9948595abb0193a472998dd5ee23f2d4d43b73bbc169", size = 1459332, upload-time = "2026-08-29T00:22:34.209Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f4/583088cbf9333263fffa8c4169e81785deca9b1146d5e38e6625197334b1/dulwich-1.2.14-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ad8b6c0ec611170af13c0ddac0b3191a20dfcd69360a06df7e60faaae14c5b29", size = 1490883, upload-time = "2026-08-29T00:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/17/01/b592e8f3b3de2cf3c3c773f521e3834cb7c8bc1b944c0c6c912e58d62460/dulwich-1.2.14-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:26440ebc7b8208edf9f251a7820a618bbdb53f1af71f2e5817bdbe3e6d24405d", size = 1459517, upload-time = "2026-08-29T00:22:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/5c/54/3ddaada9e16238a5f0103262ba79904a48cc87fc63300fb93a7417fc2873/dulwich-1.2.14-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e0f583a85a40a38400ce4374edccc2e6a4a5c5f34d4ab272b1d2d4fd3358944d", size = 1490305, upload-time = "2026-08-29T00:22:51.761Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c5/86e473a7187fae33e3f3d478e72e18166a5a986275c8c6307e7198c96d8a/dulwich-1.2.14-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a839358029e679c1f18a1fcce847d48ed9b0df7469310c72a3878c4e167447ee", size = 1461994, upload-time = "2026-08-29T00:23:04.171Z" }, + { url = "https://files.pythonhosted.org/packages/80/b2/d56697144814152a6c8c7e94ba9ee28d180acf0c56b22a734004d8a727f4/dulwich-1.2.14-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:26a36829b8590c2c431d86acc6ec4d95d708a5efbc3df0e6cb151fa8baa7b607", size = 1492031, upload-time = "2026-08-29T00:23:06.09Z" }, + { url = "https://files.pythonhosted.org/packages/70/63/ca17bbd708b56687093d7e9278944908f1471e72e95b5e97848d96619675/dulwich-1.2.14-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ecdde801f145fa4cccf87914b61d34b3ba8e70b3c7e054447c07e119eeb48f43", size = 1458574, upload-time = "2026-08-29T00:23:15.906Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c3/627bd8e13bc55476967ac9849b3b040596b4e9b012558abaf960698c55d8/dulwich-1.2.14-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:404d6b78d2012299eff296f1baa808b03cf1c04dcce0972efe79005bac1861a0", size = 1539905, upload-time = "2026-08-29T00:23:17.758Z" }, + { url = "https://files.pythonhosted.org/packages/18/eb/951fa6624e50b772bbbb15a75bc5669ae954d08170dd7b72f8e51fa33cba/dulwich-1.2.14-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ca4a6bf4ca6d9a0025523284ce82a89d6b709684c3cb25a896f407c4a55d51bc", size = 1462641, upload-time = "2026-08-29T00:23:30.469Z" }, + { url = "https://files.pythonhosted.org/packages/45/c4/e219b9e16452843b54caa98accf42bb8af493706bba292ef06ba1871cfad/dulwich-1.2.14-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:26d36df85e4247427616ad2e4aa7698adba55ee6d8c83fa5dbe93aacc2250f2f", size = 1540358, upload-time = "2026-08-29T00:23:32.517Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9b/f7a5536d8d810c90ad7d337af763e2afd713f4d2237deac34fade3ad1b50/dulwich-1.2.14-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:89aa99ae03475f12a7e66c4f649cf4b653e457b63dca1acc7b7579ca3d7aed51", size = 1459927, upload-time = "2026-08-29T00:23:42.301Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5f/13e07d1a1ab152dcbe0433791f1aed66c8b21790270ee3e7424c3559dc98/dulwich-1.2.14-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:2e5b14e5d01b7020a39acad4d7990c5da379ac0cba08d1d7121f3d036dc80983", size = 1493129, upload-time = "2026-08-29T00:23:44.198Z" }, + { url = "https://files.pythonhosted.org/packages/c2/45/5c62eeb71c6005fa4f1b5101def14271eb1063720fefcde17bcc95c0fb42/dulwich-1.2.14-py3-none-any.whl", hash = "sha256:3e1b6e13e23f9bc633fd249563e4d7a946d7d470f210967853ae0991589edd11", size = 730142, upload-time = "2026-08-29T00:23:49.691Z" }, +] + +[[package]] +name = "dynesty" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/75/db494edc8abfe2273b9cb08c87a2a1165c6ae90d76f746980ef07ec5303a/dynesty-3.1.0.tar.gz", hash = "sha256:851717431f04f749bca45e86704c28fecf9f3b10f85a2a69820b26e42cefab68", size = 35564761, upload-time = "2026-07-17T14:38:31.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/f9/41c72d0b5fb511fdf23cf62ed50da6c73d4497f03ed1cad04f214dc477c8/dynesty-3.1.0-py3-none-any.whl", hash = "sha256:4e5b77bb261abdcb98e3a538d7498a94d5882a848f0ea46924b404396ef036d5", size = 106027, upload-time = "2026-07-17T14:38:28.523Z" }, +] + [[package]] name = "emcee" version = "3.1.6" @@ -873,6 +933,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "fast-pt" +version = "3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/11/83df1a1acc4c346f35219481f464f0264932ac3c57f053b1ce68f4d13d02/fast_pt-3.2.tar.gz", hash = "sha256:ec4cfdb4a352b300ca7ec2f313a73c5676ec61ef665345b0fc37a258f5ecbd5d", size = 37099, upload-time = "2024-11-01T00:45:36.156Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/64/db25374b45218b0d305a4378a7fe8399406989e4a26f7cbafc4930200d5b/fast_pt-3.2-py3-none-any.whl", hash = "sha256:c988e312bc79df66f1719fcb198ab984fd614a6c942f1e3af6adb07180760387", size = 49051, upload-time = "2024-11-01T00:45:34.544Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.2" @@ -2267,6 +2341,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/dc/f3dfb7488b770f3f67e6545085bf2abea5172e88f57b8ad25ef860ca704c/myst_parser-5.1.0-py3-none-any.whl", hash = "sha256:9c91c52b3cdb4d94a6506e4fab4e2f296c7623a0da0dcbe6de1565c3dad67a8a", size = 85817, upload-time = "2026-05-13T09:38:17.904Z" }, ] +[[package]] +name = "narwhals" +version = "2.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/7b/6248dada39781db1ab3ebf08943080df0796098515a87f6f8696d14ec744/narwhals-2.25.0.tar.gz", hash = "sha256:62c036c810662bf7820b7737077176313bc59350eeeefb808510f388c743e4b2", size = 677076, upload-time = "2026-08-20T18:10:15.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/55481808fd70ef1567cf13540ffd4702af3f74b112e35427564b03f79c2d/narwhals-2.25.0-py3-none-any.whl", hash = "sha256:1f0f403e8c7e4463cde9bfe78b12fdd809e3ae3dda6d9b2f802934fb9c7a6a8f", size = 467373, upload-time = "2026-08-20T18:10:13.834Z" }, +] + +[[package]] +name = "nautilus-sampler" +version = "1.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/2c/a50c1a550ba43d86a0b41cf6410986d478d8640139c8e963f4dde8a8bfc2/nautilus_sampler-1.0.6.tar.gz", hash = "sha256:4e90b6d97be742be2e255c35a79e5a388cee043b4d2ce970eec327ed34139b78", size = 43118, upload-time = "2025-12-29T14:08:25.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/54/372c42bcfe324b650946f6cfee0e5adbd55aafc25cab555460bf5d9e8d36/nautilus_sampler-1.0.6-py3-none-any.whl", hash = "sha256:de6f3b9d249d87f05a673e8597e56621037e84dc712a6c2d3c18ff3672735b9b", size = 35772, upload-time = "2025-12-29T14:08:24.603Z" }, +] + [[package]] name = "nbclient" version = "0.11.0" @@ -2758,6 +2856,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-bobyqa" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pandas" }, + { name = "scipy" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/41/c4c74daf208ed27e14071e92efb7ea238ffdf77ee93a3a7777ff02d2b0e4/py_bobyqa-1.5.0.tar.gz", hash = "sha256:3c7719b68b28834ea6d538f54603f6a891263f7c21f1a673de79e3a5e0e7e413", size = 51486, upload-time = "2024-09-16T03:52:32.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/1e/0d44a4e3a291c009a357fbd1d61511d9306c2c4db9a7ceb6e8104d8d385f/Py_BOBYQA-1.5.0-py3-none-any.whl", hash = "sha256:457afc04d6f2c9f1814934854dc4e542c5e5982a0f80add4b211fcdb0b5811e3", size = 57978, upload-time = "2024-09-16T03:52:30.667Z" }, +] + [[package]] name = "py-cpuinfo" version = "9.0.0" @@ -3354,6 +3467,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/9f/56288a8aabafa24679d32d01e7db732e8e89e4074cc91aedc0bb7b7c8e46/sacc-2.4-py3-none-any.whl", hash = "sha256:da6b648998738c7c307cbdff644429633726844f8c048642725be5efc1859fa3", size = 50931, upload-time = "2026-07-02T11:29:26.286Z" }, ] +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/ee/5adbc77656b71f9456a2f5a7a9fdb4bcf9207a6b962889f1c2f9323afa4e/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f", size = 8837603, upload-time = "2026-06-02T11:53:30.328Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, + { url = "https://files.pythonhosted.org/packages/0c/39/ffe829a5b8ecb40a518724a997794657fdc354ada5e8fe8e64d998c0bac9/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38c3dcb9a1ffb85505ec53d54c7b4aea0cff70050425a7760c2af661ac85df05", size = 8789690, upload-time = "2026-06-02T11:53:47.461Z" }, + { url = "https://files.pythonhosted.org/packages/1f/88/8dab5de10c638c083772a6be83a3d8106ced492f74a928c8693638e5bb50/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da76d09304a4706db7cc1e3ebaa3b6b98a67365cc11d2996c4f1e58ba47df714", size = 9087723, upload-time = "2026-06-02T11:53:50.702Z" }, + { url = "https://files.pythonhosted.org/packages/7d/79/f4a0c4fe9711154cddabf913471153af79056382ddc612cfe5ee0ff4b72e/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5162ad10a418c8a282dde04c9aa06965de3e9a65f33c1440c0ae69bb1a09d913", size = 8847671, upload-time = "2026-06-02T11:54:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/f0/af/4d72d9e475ac83719160c662619e4bf7b95c19507cd582e7d0167a3c3dae/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fea2cc5677ab49d6f5bade978c866da44957b712d92e9635e8b4f723013c3cb", size = 9118104, upload-time = "2026-06-02T11:54:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/18/3c/e440e039bb82cd19004edaaad00acbde0fb9b461083c3ecf37941c557312/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:147e9329ef0e39f75d4cffa02b2aa48d827832684926cd5210d9a2cb5c57246b", size = 8855050, upload-time = "2026-06-02T11:54:21.699Z" }, + { url = "https://files.pythonhosted.org/packages/43/26/b341b8dab5998da6270a3a42c2152c578501354d36f944b5856757035ef8/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bad8f8b9950321b54c965fdcbac6c6c55e79e16646b49977bcf3668d3870a1a", size = 9097190, upload-time = "2026-06-02T11:54:24.454Z" }, +] + [[package]] name = "scipy" version = "1.17.1" @@ -3749,6 +3885,8 @@ test = [ { name = "ruff" }, ] workflow = [ + { name = "cosmosis" }, + { name = "fast-pt" }, { name = "mpi4py" }, { name = "snakemake" }, ] @@ -3762,9 +3900,11 @@ requires-dist = [ { name = "colorama" }, { name = "cosmo-numba", git = "https://github.com/aguinot/cosmo-numba.git?rev=main" }, { name = "cosmology", marker = "extra == 'glass'", specifier = "==2022.10.9" }, + { name = "cosmosis", marker = "extra == 'workflow'", specifier = ">=3.25" }, { name = "cryptography" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util.git?rev=develop" }, { name = "emcee" }, + { name = "fast-pt", marker = "extra == 'workflow'", specifier = ">=3.2,<4" }, { name = "fitsio", marker = "extra == 'glass'" }, { name = "getdist", git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5" }, { name = "glass", marker = "extra == 'glass'", specifier = "==2025.1" }, @@ -4422,6 +4562,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/0a/469e2bd01be1490336e6c8707386845655d59261543315778a3ccc7e8019/zarr-3.2.1-py3-none-any.whl", hash = "sha256:f78cdd3d9687ad0e9f9cba2c5683b64f0c52589c19f685eeabe872e93cc0d2c7", size = 319617, upload-time = "2026-05-05T12:37:20.66Z" }, ] +[[package]] +name = "zeus-mcmc" +version = "2.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pytest" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "seaborn" }, + { name = "setuptools" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/c0/248a600ae0f3d1437612821f7c528c895b6a8408a052c352d4cac46dcf94/zeus-mcmc-2.5.4.tar.gz", hash = "sha256:594baa90de4ad4488c4db5ed6a0446f7103bc4b3de787f4d7d23c91c9aa88769", size = 35051, upload-time = "2023-01-12T04:14:20.688Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/cb/99ee8021fa305d65b1fb369fabf96ccc65c60282407390e5355bb9e62f9f/zeus_mcmc-2.5.4-py3-none-any.whl", hash = "sha256:a64a7dae15f413200c6d590a3edfc3b8bc63c6bea3acdf123c0397c8089e123f", size = 24122, upload-time = "2023-01-12T04:14:19.24Z" }, +] + [[package]] name = "zipp" version = "4.1.0" diff --git a/workflow/README.md b/workflow/README.md index f20ffe8f..4171074e 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -51,9 +51,231 @@ Give the target *before* `--configfile`: `--configfile` takes one-or-more paths, so a target after it is read as a config file ("No such file: im_mbias"). Always dry-run first with `-n`. -The profile carries only cluster policy — no container settings (the image-sims -rules own their `apptainer exec` call) and no `OMP_NUM_THREADS` (pinned to 1 at -that same `apptainer exec` line, since the slurm executor's `--export=ALL` -propagates the driver's env, not a profile flag). Per-rule `mem_mb` / `runtime` -stay on the rules. Off-cluster, drop `--profile` and add `-j N`. See the -profile's own comments for the full rationale. +Every rule runs inside the sp_validation container: the profile sets +`software-deployment-method: apptainer` and `apptainer-args` (the bind mounts), +and Snakemake wraps each job's `shell:`/`script:` command in `apptainer exec` +itself — no rule writes its own `apptainer exec` call. The image name comes +from the `container:` directive in `workflow/Snakefile` (or a rule's own +override, e.g. the image-sims `SIF`). + +A few rules shell out to a host toolchain (CosmoCov, ImageMagick) and keep +`container: None`; each says why in its own docstring. + +`OMP_NUM_THREADS` is not set by the profile either: the slurm executor's +`--export=ALL` propagates the driver's env, not a profile flag, so a rule that +needs it pinned sets it itself. Per-rule `mem_mb` / `runtime` stay on the rules. + +### Off candide — the default profile + +Candide is where the analysis runs, so the candide profile is the one to reach +for. `profiles/default/config.yaml` exists for anywhere else — a laptop, another +cluster — and carries the machine-independent half (container wrapping, rerun +triggers, latency wait) with no SLURM layer: + +```bash +snakemake --profile workflow/profiles/default -s workflow/Snakefile \ + --configfile -j 4 +``` + +Snakemake cannot compose profiles, so both files carry that block (marked +`GENERIC` in each) — change one, change the other. `apptainer-args` is not part +of it: expect to edit the default profile's `--bind` list for your machine. + +### Which `sp_validation` a rule imports: the launched checkout + +The image is the frozen *dependency stack*; the `sp_validation` that runs is +the one in the checkout you launched from — `common.configure()` puts that +checkout's `src/` on each job's `PYTHONPATH` +(`common.inject_checkout_pythonpath` has the mechanics). + +This is the default because the alternative is incoherent: Snakemake's +`script:` directive already runs the checkout's *script files*, so without it a +rule executes new script code against an old `import sp_validation` — the two +halves of one commit, split. + +**Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not +`src/`. Editing a module under `src/` does not by itself mark outputs stale — +force with `-F` or `--forcerun `. + +To reproduce a run from the image alone, opt out: + +```bash +snakemake --profile workflow/profiles/candide --config checkout_pythonpath=false +``` + +Either way the checkout has to sit under one of the profile's bind mounts to be +visible inside the job. + +Most of the time this default is all you need. Reach for a different *image* +only when the dependency stack changed — a new package, a lockfile bump — not +when only `src/` did; "Running an image other than your own" below has the +override. + +### Never write `/automnt/nXXdataN` in a path + +Use the plain form `/nXXdataN/...` in every rule, config, and invocation +directory. `/automnt/nXXdataN` works only from a node that does *not* own that +disk. On the owning node the disk is mounted directly at `/nXXdataN` and there +is no `/automnt/nXXdataN` entry at all, so a job that lands there dies about one +second after the allocation starts, before any log file is written. This is why +`n17` is in the profile's exclude list. Every canonical path in `common.py` +already uses the plain form; keep new paths the same. + +### Run Snakemake from the host, never from inside the container + +`snakemake` is a thin host-side tool, pinned once per machine: + +```bash +uv tool install snakemake==9.23.1 --with snakemake-executor-plugin-slurm +``` + +(match the version to `snakemake` in this repo's `uv.lock`). Run every +`snakemake` command directly on the host — do not `apptainer shell` first. +Snakemake itself never touches the science stack; it only reads rule +definitions and submits jobs. Each job carries its own `apptainer exec` +wrapping from the profile (see above), so the container is where the science +code runs, not where the orchestrator runs — one container per job, never a +nested one. + +Check for a stray `~/.local/bin/snakemake` (any host-side `pip install --user +snakemake` leaves one): Apptainer passes your `PATH` and mounts your `$HOME` by +default, so it can silently shadow the one `uv tool install` set up. `which +snakemake` should resolve under `uv tool dir`, not `~/.local/bin`. + +### The container image — one per person + +Everything runs one image, published by CI as a registry tag: + +``` +docker://ghcr.io/cosmostat/sp_validation:develop +``` + +**Each person keeps their own copy of it.** There is no shared image directory: +you pull your own file, you refresh it when you want to, and nobody else's +refresh moves the ground under your running jobs. The canonical path is + +``` +~/.cache/sp_validation/sp_validation.sif +``` + +and a small CLI, `spv-container`, is what puts it there and tells you about it: + +```bash +spv-container pull # fetch :develop to the canonical path (~1.5 GB / ~15 min) +spv-container status # what is here, which commit it was built from, how current +spv-container exec # run something inside it, candide binds already applied +``` + +It ships as a console script with the package, and — being stdlib-only, because +it has to run on the *host* — also works straight from a checkout with nothing +installed: run `python3 src/sp_validation/container.py`, or put it on your PATH +once (the README's install step): + +```bash +ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container +``` + +**Do the pull from a compute node**, not the login node: it moves ~1.5 GB and +takes about fifteen minutes. `pull` writes to a temporary name and renames, so a +job either gets the whole old image or the whole new one; jobs already running +hold the old file open and finish against it unharmed. + +```bash +salloc -p comp -c 4 --time=01:00:00 --exclude=n17,n09,n36 --no-shell # note the job id +srun --jobid= spv-container pull +scancel +``` + +**How the workflow finds it.** One resolution order, shared by the CLI and the +workflow: your **sandbox** if you have built one (below), else your **`.sif`** if +you have pulled one, else the **registry tag** — which Snakemake autopulls into +`.snakemake/singularity` under the working directory. That works, but re-pulls +once per run directory, so `spv-container pull` is the path to prefer. Snakemake +accepts all three forms, a sandbox directory included. The tag itself is written +down once, as +`CONTAINER_URI` in `sp_validation/container.py`, which `workflow/common.py` +re-exports; the image-sims `sif:` config key defaults to `null` and resolves the +same way. Override any of it with `--config container=...` (below), or point +somewhere else entirely with `SPV_CONTAINER`. + +At launch the workflow prints one advisory line if your image was built from a +commit behind your checkout. It never fails the run — an older image is normally +fine, since the checkout's `src/` is what rules import (see above). It matters +when the *dependency stack* moved: a new package, a lockfile bump. + +#### When you need to install something: the sandbox + +The pristine SIF is read-only, which is what you want almost always — it is +exactly the published image, and two people running it run the same thing. But +mid-analysis you sometimes need a package the image does not carry yet, and +rebuilding through CI to find out whether it helps is too slow a loop. + +For that, unpack the image into a writable directory once: + +```bash +spv-container sandbox # ~/.cache/sp_validation/sandbox/ +spv-container exec --writable pip install +``` + +Writes into a sandbox persist. It is opt-in — nothing builds one for you — and +once it exists **it takes precedence over the SIF everywhere, workflow jobs +included**, so a package you install this way is available to your Snakemake runs +without any further wiring. Jobs exec it read-only; only `--writable` writes. + +The cost is that what you are running is no longer fully described by a revision +label. `spv-container status` says which layer is live and flags that, and the +workflow prints one line at launch when a sandbox is in play — the divergence is +visible, never silent. When you are done exploring, either fold the dependency +into `pyproject.toml` (the real fix) or reset to a clean image: + +```bash +spv-container pull # refresh the pristine SIF +spv-container sandbox --force # discard the sandbox, rebuild from it +``` + +**Where the image comes from.** CI (`.github/workflows/deploy-image.yml`) builds +it on every push, `FROM ghcr.io/cosmostat/shapepipe:im_sims` with `uv sync +--frozen` against `uv.lock`, and publishes to `ghcr.io/cosmostat/sp_validation` +tagged by branch — so `:develop` tracks the tip of `develop`. The package is +public; no credentials are needed. Your pulled file is a *snapshot*: CI +publishing a new image changes nothing until you pull again. + +`spv-container status` reads `org.opencontainers.image.revision` — the +sp_validation commit the image was built from — and places it against your +checkout's `HEAD`, naming which layer (sandbox or SIF) it read. The image-sims workflow records the same label in +`m_bias_config.yaml` as `ghcr_revision`, so a result file says which image +produced the number. + +#### Running an image other than your own + +`container` is a config key read by every entry Snakefile +(`common.resolve_container`), so one flag overrides the default everywhere at +once — a local file, or any CI tag: + +```bash +snakemake --profile workflow/profiles/candide --config container=/path/to/my.sif +snakemake --profile workflow/profiles/candide \ + --config container=docker://ghcr.io/cosmostat/sp_validation:my-branch +``` + +CI tags an image for **every** branch, by sanitized branch name (`/` → `-`), so +the second form is how you test a branch's own stack. Snakemake autopulls a tag +per run directory (~15 minutes — from a compute node); `spv-container pull --tag +` instead puts it at your canonical path, where it becomes your default. + +For the image-sims workflow, set `image_sims: {sif: ...}` in your run config. +Either way the image has to sit under one of the profile's bind mounts to be +visible. + +One trap to know: the `script:` directive bind-mounts the host orchestrator's +`snakemake` into the job and *appends* it to `sys.path`, so a `snakemake` +importable inside the image wins the lookup. If `script:` rules start failing +with `ModuleNotFoundError: No module named 'snakemake.iocontainers'` or similar, +an in-image snakemake older than the host's is the first thing to check. + +### `snakemake` in `script:` files + +Every script run via a rule's `script:` directive uses a bare `snakemake` +name (`snakemake.input[...]`, etc.) with no import — Snakemake injects it as +a module global before the script runs. `from snakemake.script import +snakemake` is IDE-hint-only and raises `ImportError` if actually executed. diff --git a/workflow/Snakefile b/workflow/Snakefile index 7d91db81..fe05f02b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -8,19 +8,15 @@ # or run standalone with --configfile pointing at a paper config # (e.g. papers/bmodes/config/config.yaml). -container: "/n17data/cdaley/containers/containers" - envvars: "PYTHONUNBUFFERED", import os import sys -# Host OpenMPI libs for MPI rules inside the container. -# Apptainer passes APPTAINERENV_* vars into the container as their unprefixed names. -# Only affects rules that import mpi4py; safe for all other rules. -os.environ["APPTAINERENV_LD_LIBRARY_PATH"] = "/softs/openmpi/5.0.5-slurm-CentOS8/lib" - +# Machine-specific env belongs to the machine, so it rides the profile's +# `apptainer-args --env`, not this file. See workflow/profiles/. +# # Shared helpers live in common.py next to this Snakefile. Snakemake's `module` # imports rules, not Python globals, so helpers travel by plain Python import; # when composed, the paper Snakefile has already imported common and this hits @@ -28,9 +24,14 @@ os.environ["APPTAINERENV_LD_LIBRARY_PATH"] = "/softs/openmpi/5.0.5-slurm-CentOS8 sys.path.insert(0, os.path.realpath(str(workflow.basedir))) import common +# configure() also puts this checkout's src/ on the container's PYTHONPATH -- +# see common.inject_checkout_pythonpath. common.configure(config) from common import * +# The one image every rule runs in; see common.resolve_container. +container: common.resolve_container(config.get("container")) + # Wildcard constraints — centralized in common.py, not in individual rule files wildcard_constraints: **WILDCARD_CONSTRAINTS diff --git a/workflow/common.py b/workflow/common.py index 3df3413b..506f7cb2 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -1,10 +1,36 @@ """Shared helpers for the B-modes Snakemake workflow.""" +import importlib.util import json import os import re +import sys from pathlib import Path +# This checkout's importable source tree: workflow/common.py -> /src. +REPO_SRC = Path(__file__).resolve().parent.parent / "src" + +# The container model lives in the package (``sp_validation/container.py``). +# Taken from *this checkout's* src/, so the workflow and the ``spv-container`` +# CLI can never disagree about which image to run. +# +# Loaded by file path rather than as ``sp_validation.container``: snakemake runs +# on the host, where sp_validation is usually not installed, and importing the +# package would drag in ``__init__`` -> ``version`` -> a metadata warning on +# every launch. The module itself is stdlib-only, so this costs nothing. +_container = importlib.util.module_from_spec( + importlib.util.spec_from_file_location( + "_spv_container", REPO_SRC / "sp_validation" / "container.py" + ) +) +sys.modules["_spv_container"] = _container +_container.__loader__.exec_module(_container) + +compare_revision = _container.compare_revision +image_revision = _container.image_revision +resolve_image = _container.resolve_image + + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( @@ -50,9 +76,90 @@ PLANCK18 = None +def inject_checkout_pythonpath(workflow_config): + """Make the launched checkout's ``src`` win over the image's baked copy. + + Snakemake's ``script:`` directive already runs the *checkout's* script + files, so without this a rule executes new script code against an old + ``import sp_validation`` -- the two halves of one commit, split. Prepending + ``REPO_SRC`` closes that: the image stays the frozen dependency stack, the + checkout supplies sp_validation. + + Apptainer forwards ``APPTAINERENV_``-prefixed host variables into the job as + their unprefixed names, surviving the profile's ``--cleanenv``; setting it + here on the driver reaches every containerized rule. Any value the user + already exported is preserved behind ours. + + Opt out with ``--config checkout_pythonpath=false`` to reproduce a run from + the image alone. + """ + flag = workflow_config.get("checkout_pythonpath", True) + # `--config key=false` can arrive as the *string* "false" depending on how + # Snakemake parses the value, so don't lean on truthiness alone. + if isinstance(flag, str): + flag = flag.strip().lower() not in ("false", "no", "0", "off", "") + if not flag: + return + if not REPO_SRC.is_dir(): + return + existing = os.environ.get("APPTAINERENV_PYTHONPATH", "") + parts = [str(REPO_SRC)] + [p for p in existing.split(":") if p] + os.environ["APPTAINERENV_PYTHONPATH"] = ":".join(parts) + + +def resolve_container(override=None): + """Return the image every rule should run in. + + ``override`` wins if set (a ``docker://`` tag, a ``.sif`` path or a sandbox + directory -- Snakemake's ``container:`` accepts all three); otherwise + ``resolve_image()``, so jobs run what interactive ``spv-container`` work + runs. + """ + if override: + return str(override) + return resolve_image()[0] + + +def warn_if_image_stale(): + """Print one advisory line about a local image that is not pristine or current. + + Never fatal. Two things worth saying at launch: + + * a sandbox is in play, so what jobs run is not fully described by any + revision label -- deliberate, but it should not be a silent difference + from a clean run; + * the image predates the checkout. Usually fine, because the checkout's + ``src/`` is what rules import; it matters when the *dependency stack* + moved -- a new package, a lockfile bump. + + Silent when there is no local image, no apptainer, or no revision label. + """ + image, kind = resolve_image() + if kind == "tag": + return + revision = image_revision(image) + if kind == "sandbox": + built = f"built from {revision[:12]}" if revision else "revision unknown" + print( + f"[container] running the writable sandbox at {image} ({built}). " + "Anything installed into it is part of this run; " + "`spv-container status` for detail.", + file=sys.stderr, + ) + if compare_revision(revision) == "behind": + print( + f"[container] image was built from {revision[:12]}, which is behind this " + "checkout. Fine unless the dependency stack moved; refresh with " + "`spv-container pull`.", + file=sys.stderr, + ) + + def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + inject_checkout_pythonpath(workflow_config) + warn_if_image_stale() CATALOG_CONFIG = workflow_config FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile index d7a89065..7420ae68 100644 --- a/workflow/image_sims/Snakefile +++ b/workflow/image_sims/Snakefile @@ -4,38 +4,37 @@ Run the sp_validation-side chain (merge -> extract -> calibrate -> m-bias), optionally including the ShapePipe pipeline stage, without pulling in the cosmology-validation config the top-level ``workflow/Snakefile`` requires. -Layer a run config over the operational defaults -- the workflow config.yaml -carries operational defaults but *no* science keys, so it is incomplete on its -own (by design); the run config supplies the science knobs. The one drive -command on candide, with the committed SLURM profile owning all scheduling: +The one drive command on candide, with the committed SLURM profile owning +scheduling and container wrapping: snakemake --profile workflow/profiles/candide \\ -s workflow/image_sims/Snakefile \\ im_mbias --configfile my_run.yaml -``configfile: "workflow/image_sims/config.yaml"`` below loads the operational -defaults automatically, so only ``my_run.yaml`` (the science knobs, and any -operational override that run wants) is passed on the command line; Snakemake -deep-merges the two. The profile supplies the executor, account, partition, -node excludes and job floor -- no ``-j`` needed (the slurm executor sets the -job cap). Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. - -The target (``im_mbias``) is given *before* ``--configfile``: Snakemake's -``--configfile`` takes one-or-more paths, so a target placed after it is -swallowed as a config path ("No such file: im_mbias"). Put targets ahead of -``--configfile`` (or make ``--configfile`` the last flag on the line). Always +``configfile:`` below loads the operational defaults, so ``my_run.yaml`` need +only carry the science knobs (and any operational override); Snakemake +deep-merges the two. Off-cluster, drop ``--profile`` and add ``-j N``. + +Put targets *before* ``--configfile``: it takes one-or-more paths, so a target +after it is swallowed as a config path ("No such file: im_mbias"). Always dry-run first with ``-n``. -The same rules are also available inside the main workflow: they are included -there under ``if "image_sims" in config``. +The same rules are also included in the main workflow under +``if "image_sims" in config``. """ -configfile: "workflow/image_sims/config.yaml" +import os +import sys +# Shared helpers from the generic workflow one directory up -- resolve_container +# in particular, so `sif: null` here falls back to the same image every other +# entry point runs. Snakemake's `include:` shares Python globals, so +# image_sims.smk sees this import. +sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), ".."))) +import common -# The image-sims rules own their container invocation explicitly, so no -# top-level container is needed here. -container: None + +configfile: "workflow/image_sims/config.yaml" include: "../rules/image_sims.smk" diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 8f719c68..f5f3e3c0 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -1,44 +1,31 @@ # Image-simulation m-bias workflow configuration. # -# Two kinds of keys live under `image_sims:`, and the split is the point: -# -# * OPERATIONAL keys default here (active lines below) and *nowhere else* -- -# the .smk reads them bare, so this file is their single home. Override in -# a run config only when a run genuinely differs from the shared setup. -# -# * SCIENCE keys have NO default -- not here, not in code. They fix the -# estimator's scientific behaviour and must be stated per run, so they -# appear below only as commented template lines. Supply them in a run -# config layered on top: -# -# snakemake -s workflow/image_sims/Snakefile \ -# --configfile workflow/image_sims/config.yaml \ -# --configfile my_run.yaml \ -# -j 4 im_mbias -# -# A run config that omits a science key fails at DAG parse, naming the key; an -# unknown key under `image_sims:` fails as a typo. The structural keys below -# (sif, repos, data roots, num, tile_ids) also have no default and must be set. +# OPERATIONAL keys default here and nowhere else -- the .smk reads them bare, so +# this file is their single home. SCIENCE keys have no default anywhere and +# appear below only as commented template lines: they fix the estimator's +# behaviour, so each run must state them in a run config layered on top. A run +# config that omits one fails at DAG parse, naming the key; an unknown key under +# `image_sims:` fails as a typo. image_sims: - # --- containers ------------------------------------------------------- - # Two images, one per half of the chain (the split gate766 ran). One image - # is the eventual target -- the sp_validation image is FROM the ShapePipe - # image -- but until sp_validation is uv-locked with cosmo_numba declared, - # its published image can drift NumPy past numba's window (seen 2026-07-11: - # "Numba needs NumPy 2.4 or less. Got NumPy 2.5" at ngmix). PYTHONPATH - # shadows pure-Python code only, never binary deps. - sif: /n17data/cdaley/containers/sp_validation_im_sims.sif # extract/calibrate/m-bias - sif_pipeline: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif # pipeline/merge - # Apptainer bind mounts. /automnt is required when repos/data are - # automounted (candide gotcha); harmless otherwise. [operational] - binds: /n17data,/n09data,/home,/automnt + # --- container -------------------------------------------------------- + # One image for the whole chain: the sp_validation image is built FROM the + # ShapePipe image, so it carries both stacks. CI builds it from the uv lock -- + # an unlocked build drifts NumPy past numba's ceiling and the ngmix stage dies + # ("Numba needs NumPy 2.4 or less"). + # + # `null` means the workflow's one image: your own .sif if you have pulled one + # with `spv-container pull`, else the registry tag for Snakemake to autopull + # (workflow/README.md). Neither is repeated here. + # Override with a local .sif path, or with a branch tag + # (docker://ghcr.io/cosmostat/sp_validation:) to run a + # branch's own CI image. + sif: null # --- repositories ----------------------------------------------------- # Bound into the image; both repos' src go on PYTHONPATH so this branch's - # code wins over the baked copies: ShapePipe's #766 build, and sp_validation's - # image_sims.py / catalog.match_catalogs_radec. + # code wins over the baked copies. shapepipe_repo: /n17data/cdaley/unions/code/shapepipe sp_validation_repo: /n17data/cdaley/unions/code/sp_validation diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 0316b909..19afd89a 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -1,71 +1,60 @@ # Committed SLURM profile for the candide cluster (IAP). # -# This is the "one run command" half of the workflow: drive any target with +# Drive any target with # # snakemake --profile workflow/profiles/candide \ # -s workflow/image_sims/Snakefile \ # --configfile # -# and Snakemake owns all scheduling -- it fans out one SLURM job per branch x -# tile and drives them against the cluster, MPI-free. Everything here is -# cluster policy (executor, account, partition, node excludes, per-job -# defaults); it carries no science and no workflow logic. -# -# What is deliberately NOT here: -# -# * Container / apptainer settings. The image-sims rules set -# ``container: None`` and own their ``apptainer exec`` call through the -# shared ``EXEC`` prefix (one image for every stage, with PYTHONPATH / -# PSF_DICT / OMP_NUM_THREADS injected there). So no -# ``software-deployment-method: apptainer`` / ``apptainer-args`` -- those -# would wrap a *second*, redundant container around jobs that already run -# inside one. -# -# * OMP_NUM_THREADS. It is pinned to 1 on the ``apptainer exec`` line in -# workflow/rules/image_sims.smk, not here. The slurm executor submits with -# ``--export=ALL``, which propagates the *driver's* ambient environment; a -# profile only sets CLI flags, never the driver's own env, so an -# ``OMP_NUM_THREADS`` set here would silently depend on the operator having -# exported it by hand. Injecting it at the container boundary puts it where -# the compute runs, committed and independent of the launching shell. -# -# * Per-rule resources (mem_mb, runtime). Those live on each rule in the -# .smk; the ``default-resources`` below are only the floor for rules that -# set none. +# Snakemake owns scheduling and the container wrapping; run it host-side, never +# inside an ``apptainer shell``. workflow/README.md is the full story. executor: slurm -# Cluster policy applied to every job unless a rule overrides it. The excludes -# are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, -# n36); ``slurm_extra`` is passed verbatim onto the sbatch line by the executor -# plugin, so the quoting is what sbatch must see. +# --- GENERIC: mirrored in workflow/profiles/default/config.yaml ------------- +software-deployment-method: apptainer + +# Rerun a job when its code / params / inputs change, not only on mtime. +# Caveat: "code" watches rule bodies and ``script:`` files, not ``src/`` -- +# editing a module under src/ does not mark outputs stale on its own. +rerun-triggers: ["mtime", "params", "input", "code"] + +# Give an appearing output file a moment on networked filesystems before +# Snakemake calls a job failed for a missing output. +latency-wait: 5 +# --- end GENERIC ------------------------------------------------------------ + +# No ``apptainer-prefix``: the entry Snakefiles resolve ``container:`` to this +# user's own image path, so there is nothing for Snakemake to cache. # -# Three candide-specific SLURM lessons are baked into the values below (learned -# the hard way on the earlier hand-driven im-sims runs; see shapepipe's retired -# image_sims_pipeline/Snakefile docstring): +# candide's disks, plus the one machine-specific env var: the host OpenMPI libs +# MPI rules need to find libmpi inside the container (only rules importing +# mpi4py care; harmless for the rest). The bind list matches ``spv-container +# exec``'s default; keep the two in step. +apptainer-args: >- + --cleanenv + --bind /home,/scratch,/automnt,/n17data,/n23data1,/n09data + --env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib + +# Cluster policy applied to every job unless a rule overrides it. Excludes are +# the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, n36). # -# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's -# resource parser reads a *bare* number as SECONDS, so ``runtime: 60`` would -# silently give every job a 60-second wall clock and kill it on start. The -# quoted-with-unit form here is deliberate; keep it that way, and prefer the -# same in any ``--default-resources`` passed on the command line. (A bare -# integer in a *rule's* ``resources: runtime=720`` is fine -- snakemake -# reads rule-level numeric runtime as minutes -- the seconds trap is only +# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's +# resource parser reads a bare number as SECONDS, so ``runtime: 60`` would +# silently give every job a 60-second wall clock and kill it on start. +# (A bare integer in a rule's own ``resources: runtime=720`` is fine -- +# rule-level numeric runtime is read as minutes; the seconds trap is only # the CLI/default-resources parser.) # # * ``cpus_per_task`` is pinned to 12 to CAP JOBS PER NODE, not because a job -# needs 12 cores (the chain is MPI-free and pins ``OMP_NUM_THREADS=1`` at -# the container). candide's per-user process limit is ``ulimit -u 1200`` -# *per node*, and apptainer crashes ("can't start new thread") beyond ~4 -# concurrent jobs on a 48-core node. Requesting 12 CPUs/job holds SLURM to -# ~4 jobs per 48-core node, under the ceiling. Dropping this to 1 would let -# SLURM pack ~48 jobs onto a node and crash the compute-heavy im_pipeline -# stage (which inherits this default -- it sets mem/runtime but not cpus). +# needs 12 cores. candide's per-user process limit is ``ulimit -u 1200`` +# per node, and apptainer crashes ("can't start new thread") beyond ~4 +# concurrent jobs on a 48-core node; 12 CPUs/job holds SLURM to ~4 jobs per +# node. # -# * After launching a real fan-out, VERIFY the request actually landed: -# ``squeue -u $USER -o "%C %l"`` must show 12 (CPUs) and the wall clock you -# intended (e.g. 12:00:00 for im_pipeline). A silently-misparsed runtime or -# cpus shows up here before it wastes a queue slot. +# * After launching a real fan-out, verify the request landed: +# ``squeue -u $USER -o "%C %l"`` should show 12 (CPUs) and the intended +# wall clock. default-resources: slurm_account: "cusers" slurm_partition: "comp,pscomp" @@ -73,13 +62,7 @@ default-resources: cpus_per_task: 12 slurm_extra: "'--exclude=n17,n09,n36'" -# Give an appearing output file a moment on candide's automounted filesystems -# before Snakemake calls a job failed for a missing output, and retry a job -# once on transient node failure. -latency-wait: 5 +# Retry a job once on transient node failure, and keep the SLURM logs of +# successful jobs (candide debugging). retries: 1 - -# Keep the SLURM logs of successful jobs (candide debugging), and rerun a job -# when its code / params / inputs change, not only on mtime. slurm-keep-successful-logs: true -rerun-triggers: ["mtime", "params", "input", "code"] diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml new file mode 100644 index 00000000..bf272c03 --- /dev/null +++ b/workflow/profiles/default/config.yaml @@ -0,0 +1,39 @@ +# Machine-independent profile: the container model, and nothing else. +# +# Use it anywhere that is not candide -- a laptop, a workstation, another +# cluster's interactive node: +# +# snakemake --profile workflow/profiles/default -s workflow/Snakefile \ +# --configfile -j 4 +# +# On candide -- where the analysis actually runs -- use +# `--profile workflow/profiles/candide` instead: the GENERIC block below plus +# the SLURM executor and candide's machine layer. +# +# Requirements are the same everywhere: `apptainer` on PATH, and `snakemake` +# installed host-side (`uv tool install ...`, see workflow/README.md) -- never +# run from inside an apptainer shell. + +# --- GENERIC: mirrored in workflow/profiles/candide/config.yaml ------------- +# Wrap each job's `shell:`/`script:` command in `apptainer exec`, using the +# image named by the entry Snakefile's `container:` directive. +software-deployment-method: apptainer + +# Rerun a job when its code / params / inputs change, not only on mtime. +# Caveat: "code" watches rule bodies and `script:` files, not `src/` -- editing +# a module under src/ does not mark outputs stale on its own. +rerun-triggers: ["mtime", "params", "input", "code"] + +# Give an appearing output file a moment on networked filesystems before +# Snakemake calls a job failed for a missing output. +latency-wait: 5 +# --- end GENERIC ------------------------------------------------------------ + +# Binds are the one thing you almost certainly need to edit for your machine: +# whatever paths your inputs, outputs and checkout live under. `--cleanenv` so a +# job's environment is the image's, not your shell's. If $HOME and the working +# directory cover everything (apptainer mounts both by default), drop `--bind`. +apptainer-args: "--cleanenv --bind /home" + +# No `apptainer-prefix`, here or on candide: the entry Snakefiles resolve +# `container:` to this user's own image path (workflow/README.md). diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 49e40201..e9185428 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -11,7 +11,6 @@ def get_cat_params(version): # covariance_dir(), covariance_base(), covariance_path() defined in Snakefile -# Additional wildcard constraints defined locally for pseudo-Cl rules (line 327) # DEFAULT_MASK_SUFFIX defined in Snakefile # Footprint mask power spectra (nside=4096, from comprehensive catalog with spatial cuts only) @@ -145,6 +144,12 @@ EOF rule covariance_cosmocov: + """Run the host-compiled CosmoCov binary. + + `container: None` on purpose: CosmoCov is a host-compiled Fortran/C binary + loaded through environment-modules (`module load gcc intelpython openmpi`), + not a Python entry point the container ships. + """ input: rules.covariance_ini.output, output: @@ -203,13 +208,13 @@ rule covariance_glass_mock: seed=range(config["glass_mocks"]["seed_range"][0], config["glass_mocks"]["seed_range"][1] + 1), ), output: - xi_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/xi_covariance.npy", - cl_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/cl_covariance.npy", - combined_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_covariance.npy", - correlation_plot="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_correlation.png", - xi_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/xi_mean.npy", - cl_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/cl_mean.npy", - combined_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_mean.npy", + xi_covariance="results/covariance/glass_mock_v1.4.6/xi_covariance.npy", + cl_covariance="results/covariance/glass_mock_v1.4.6/cl_covariance.npy", + combined_covariance="results/covariance/glass_mock_v1.4.6/combined_covariance.npy", + correlation_plot="results/covariance/glass_mock_v1.4.6/combined_correlation.png", + xi_mean="results/covariance/glass_mock_v1.4.6/xi_mean.npy", + cl_mean="results/covariance/glass_mock_v1.4.6/cl_mean.npy", + combined_mean="results/covariance/glass_mock_v1.4.6/combined_mean.npy", script: "../scripts/compute_glass_mock_covariance.py" @@ -231,30 +236,21 @@ rule generate_glass_mock_rhotau_samples: mock_id="{mock_id}", output_dir="results/glass_mock_rhotau_samples", threads: 1 - shell: - """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/generate_glass_mock_rhotau_samples.py \ - --cov-tau {input.cov_tau} \ - --ref-tau {input.ref_tau} \ - --output-dir {params.output_dir} \ - --mock-ids {params.mock_id} - """ + script: + "../scripts/generate_glass_mock_rhotau_samples.py" rule covariance_process: + """Post-process a raw CosmoCov matrix into the analysis-ready form.""" input: str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") output: matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") - params: - output_stub=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed") threads: 1 - shell: - """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmocov_process.py {input} {params.output_stub} - """ + script: + "../scripts/cosmocov_process.py" def fiducial_covariance_outputs(mask_suffix=""): diff --git a/workflow/rules/glass_mock.smk b/workflow/rules/glass_mock.smk index da8cb491..243076d6 100644 --- a/workflow/rules/glass_mock.smk +++ b/workflow/rules/glass_mock.smk @@ -119,7 +119,7 @@ rule mock_cosebis_bias_test: ), xi_ref=f"{MOCK_RESULTS}/gg_glass_mock_00001_nbins=1000.fits", cov=str( - Path("/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance") + COSMO_INFERENCE / "data/covariance" / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked" / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked_processed.txt" ), diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index a4eb92d0..cc992391 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -1,43 +1,18 @@ """Image-simulation orchestration: raw SKiLLS sim images -> shear m/c bias. -This rule set drives the image-simulation validation chain end to end and is -the sp_validation-side half of the split described in +The sp_validation-side half of the split described in ``UNIONS-WL/MultiBand_ImSim#1``: ShapePipe turns the simulated tiles into -per-tile shape catalogues, then sp_validation merges, extracts, calibrates and -finally measures the multiplicative/additive shear bias. - -Two images, one prefix shape. Architecturally one image could run every -stage -- the sp_validation image is built ``FROM`` the ShapePipe image, so it -carries both stacks -- but the *published* sp_validation image's environment -is not yet trustworthy for the ShapePipe half: sp_validation has no lockfile -and does not declare its numba-bearing dependency (``cosmo_numba``), so -unpinned install layers can drift NumPy past numba's window (a 2026-07-11 -gate run hit exactly this: ``Numba needs NumPy 2.4 or less. Got NumPy 2.5`` -at the ngmix stage). PYTHONPATH shadowing covers pure-Python *code*, never -binary deps, so until sp_validation is uv-locked with its deps declared -(spun off as its own task), each half runs in its own repo's image -- the -same split the gate766 baseline ran: - -* ShapePipe stages -> ``pipeline`` (raw images -> per-tile cats) and ``merge`` - (``create_final_cat`` -> ``final_cat_{sim}.hdf5``) run in ``sif_pipeline`` - (the ShapePipe image). -* sp_validation stages -> ``manifest``, ``extract`` (-> comprehensive cat), - ``calibrate`` (-> cut cat) and ``m_bias`` (-> ``m_bias_results.yaml``) run - in ``sif`` (the sp_validation image). - -Every rule sets ``container: None`` and calls ``apptainer exec`` explicitly -through a shared prefix template (``EXEC_PIPELINE`` / ``EXEC`` -- identical -env injections, different image), because the images are not the workflow's -top-level container. Everything is parameterised under -``config["image_sims"]`` -- the two ``sif`` keys, repository roots, data roots, -the PSF dictionary, the explicit ``tile_ids`` list and the sim/calibration -knobs -- so a fresh user drives it from config alone, with no hard-coded clone -layout. Configuration is fail-fast: a schema check at load rejects an unknown -key (typo) and a missing science key (see ``workflow/image_sims/config.yaml`` -for the operational/science split). The ``PYTHONPATH`` override injects both -repos' ``src`` so the *branch* source (ShapePipe's ``#766`` build; -sp_validation's ``image_sims.py``, ``catalog.match_catalogs_radec``) wins over -whatever is baked into the image. +per-tile shape catalogues (``pipeline``, ``merge``), then sp_validation +extracts, calibrates and measures the multiplicative/additive shear bias +(``manifest``, ``extract``, ``calibrate``, ``m_bias``). + +One image runs the whole chain: the sp_validation image is built ``FROM`` the +ShapePipe image, so it carries both stacks. Everything else is parameterised +under ``config["image_sims"]`` -- repository roots, data roots, the PSF +dictionary, the explicit ``tile_ids`` list and the sim/calibration knobs -- so +a fresh user drives it from config alone. Configuration is fail-fast: a schema +check at load rejects an unknown key (typo) and a missing science key (see +``workflow/image_sims/config.yaml`` for the operational/science split). The five simulations per grid are the reference ``1z2z`` (no input shear) plus the ``+/-`` shear pairs ``1p2z``/``1m2z`` (g1) and ``1z2p``/``1z2m`` (g2); the @@ -45,24 +20,15 @@ m-bias estimator matches each to the reference by RA/Dec. """ import os -from pathlib import Path IMSIM = config["image_sims"] # --- fail-fast schema check ---------------------------------------------- -# One home for every fact: the run config carries the science knobs, the -# workflow config.yaml carries the operational defaults, and *this* block is -# where a typo or a missing knob dies -- at DAG parse, before any compute. +# An unknown key under ``image_sims:`` is a hard error (typo protection); a +# missing key is a hard error naming it. Both fire at DAG parse, before compute. # -# Every key must be declared below. An unknown key under ``image_sims:`` is a -# hard error (typo protection); a missing *science* key is a hard error naming -# the key (no silent code default anywhere). Operational keys default in the -# workflow config.yaml and nowhere else: the .smk reads them as bare -# ``IMSIM[key]`` (never ``.get`` with a second literal), so their value comes -# from config.yaml alone -- the single home for an operational default. -# -# Science keys: required from the *run* config; no default in config.yaml (only -# a commented template line) and no default in code. These fix the estimator's +# Science keys: required from the *run* config; no default here or in +# config.yaml (only a commented template line). These fix the estimator's # scientific behaviour, so they must be stated per run, never inherited. _SCIENCE_KEYS = { "w_cols", @@ -72,16 +38,14 @@ _SCIENCE_KEYS = { "bootstrap_seed", "mask_config", } -# Deprecated science keys: accepted (so a pre-``w_cols`` run config still parses -# and the estimator's back-compat path runs) but not *required* -- our configs -# state ``w_cols``. Listed here only to keep them out of the unknown-key error. +# Deprecated but still accepted, so a pre-``w_cols`` run config keeps parsing +# into the estimator's back-compat path. _DEPRECATED_KEYS = { "w_col", } -# Operational keys: default (visibly) in the workflow config.yaml; the .smk -# reads them bare, so config.yaml is their one home. +# Operational keys: default in the workflow config.yaml; the .smk reads them +# bare (never ``.get`` with a literal), so config.yaml is their one home. _OPERATIONAL_KEYS = { - "binds", "sims_type", "branches", "shape", @@ -94,7 +58,6 @@ _OPERATIONAL_KEYS = { # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { "sif", - "sif_pipeline", "shapepipe_repo", "sp_validation_repo", "grids_base", @@ -128,13 +91,13 @@ if _missing_structural: f"{_missing_structural} -- set them in the run config" ) -# --- containers ----------------------------------------------------------- -# Two images (see module docstring): the ShapePipe image for the pipeline and -# merge stages, the sp_validation image for everything downstream. Collapse -# back to one image once sp_validation's env is lock-managed. -SIF = IMSIM["sif"] # sp_validation stages -SIF_PIPELINE = IMSIM["sif_pipeline"] # ShapePipe stages -BINDS = IMSIM["binds"] +# --- container ------------------------------------------------------------ +# Every compute rule carries ``container: SIF`` rather than inheriting a +# module-level default: these rules are also included from the top-level +# workflow/Snakefile, whose module default is the cosmology image (no ShapePipe +# stack). Binds come from the driving profile's ``apptainer-args``. A null +# ``sif`` resolves to the workflow's one image (see workflow/image_sims/config.yaml). +SIF = common.resolve_container(IMSIM["sif"]) # --- repositories (bound into the image; branch code overrides) ----------- SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] @@ -181,48 +144,27 @@ CALIBRATE = IMSIM["calibrate_script"] # m-bias is *this branch's* extracted core, injected on PYTHONPATH. COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" -# --- container exec prefixes ---------------------------------------------- -# One prefix *shape* for every stage -- two instances, one per image. Three -# env injections make the on-disk branch -# code and the sim PSF win over the image's baked copies: +# --- in-command env prefix ------------------------------------------------- +# Snakemake wraps each rule's whole ``shell:`` string inside the container, so +# these ``VAR=value`` tokens land inside it. Three settings: # -# * PYTHONPATH prepends BOTH repos' ``src`` (ShapePipe first, then -# sp_validation), so Python resolves the worktree build before -# ``/app``/``/sp_validation`` -- the local-testing counterpart of the -# git-ref deps, letting the branch code run without an image rebuild. This -# covers the Python *packages* only: the bash entry points (run_job) and -# the ShapePipe/sp_validation *scripts* are still invoked at the repo paths -# resolved from config (RUN_JOB, CREATE_FINAL_CAT, EXTRACT_INFO, ...), not -# shadowed by PYTHONPATH. +# * PYTHONPATH prepends both repos' ``src`` so Python resolves the worktree +# build ahead of the copies baked into the image -- the branch's code runs +# without an image rebuild. Packages only: the bash and python entry points +# are invoked at the repo paths from config (RUN_JOB, CREATE_FINAL_CAT, +# EXTRACT_INFO, ...), not shadowed by PYTHONPATH. # * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded # via getexpanded) at this run's PSF dictionary. -# -# The SLURM env vars are stripped (``env -u ...``) so that when the ShapePipe -# pipeline stage's OpenMPI initialises inside the image it does not try to -# attach to the host SLURM launcher (cf. apptainer_noslurm.sh). The strip is -# harmless for the pure-Python sp_validation stages, so one prefix serves all. -# -# ``OMP_NUM_THREADS=1`` is injected here, at the ``apptainer exec`` call, and -# not left to the SLURM profile. The chain is MPI-free: Snakemake fans out one -# job per branch x tile and each job's parallelism is ShapePipe's own internal -# multiprocessing (``-N n_smp``), so the OpenMP/BLAS thread pool inside the -# container must be pinned to 1 to avoid oversubscription. The SLURM profile -# cannot pin it reliably: the slurm executor submits with ``--export=ALL``, -# which propagates the *driver's* ambient environment -- but a Snakemake -# profile only sets CLI flags, never the driver's own env, so an -# ``OMP_NUM_THREADS`` there would depend on the operator having exported it by -# hand (the implicit, uncommitted state the "one run command" is meant to -# retire). Injecting it on the ``apptainer exec`` line puts it where the -# compute actually runs -- inside the container, independent of the driver's -# env -- the same lever this prefix already uses for PYTHONPATH/PSF_DICT. -_EXEC_PREFIX = ( - "env -u SLURM_JOBID -u SLURM_JOB_ID -u SLURM_PROCID " - f"apptainer exec --bind {BINDS} " - f"--env PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " - f"--env PSF_DICT={PSF_DICT} --env OMP_NUM_THREADS=1 " +# * OMP_NUM_THREADS=1 rides here rather than in the SLURM profile: the chain +# is MPI-free (Snakemake fans out one job per branch x tile; in-job +# parallelism is ShapePipe's own ``-N n_smp``), so the OpenMP/BLAS pool must +# be pinned to 1 to avoid oversubscription, and a profile can only set CLI +# flags, never the driver env the slurm executor's ``--export=ALL`` +# propagates. +_ENV_PREFIX = ( + f"PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " + f"PSF_DICT={PSF_DICT} OMP_NUM_THREADS=1 " ) -EXEC = _EXEC_PREFIX + SIF # sp_validation stages -EXEC_PIPELINE = _EXEC_PREFIX + SIF_PIPELINE # ShapePipe stages JOB_MASK = sum([1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]) @@ -300,8 +242,10 @@ rule im_manifest: input_sims_base=INPUT_SIMS_BASE, sims_type=SIMS_TYPE, num=NUM, + container: + SIF shell: - "{EXEC} python {BUILD_MANIFEST} " + "{_ENV_PREFIX} python {BUILD_MANIFEST} " "--input-sims-base {params.input_sims_base} " "--sims-type {params.sims_type} --num {params.num} " "{params.branch_args} -o {output.manifest}" @@ -349,22 +293,17 @@ rule im_init: rule im_pipeline: - """Run ShapePipe on one simulated tile (ShapePipe stage). + """Run ShapePipe on one simulated tile. Delegates the module DAG to ShapePipe's own job runner; the sentinel log - marks tile completion for the merge step. This is the compute-heavy, - MPI-bearing stage. + marks tile completion for the merge step. The compute-heavy stage. """ input: - # ``params.py`` is a *tracked* output of ``im_init``, so this one input - # supplies the im_init -> im_pipeline edge. The ``cfis`` symlink the - # shell reads (via {RUN_JOB}) is created by that same im_init shell block - # as an *untracked* side effect -- no rule declares it as an output - # (snakemake will not track a symlink/directory output). Declaring it an - # input here therefore asked the DAG for a file no rule produces: on a - # fresh grids_base it aborted the build with MissingInputException before - # any job ran. It is safe to drop -- cfis exists whenever params does, - # since im_init stages both together. + # params.py alone supplies the im_init -> im_pipeline edge. The `cfis` + # symlink {RUN_JOB} also reads is an untracked side effect of the same + # im_init shell (snakemake will not track a symlink output), so it must + # not be declared here -- doing so asks the DAG for a file no rule + # produces and aborts on a fresh grids_base. params=f"{GRIDS_BASE}/{{sim}}/params.py", output: done=touch(f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done"), @@ -375,9 +314,11 @@ rule im_pipeline: resources: mem_mb=16000, runtime=720, + container: + SIF shell: "cd {params.run_dir} && " - "{EXEC_PIPELINE} bash {RUN_JOB} " + "{_ENV_PREFIX} bash {RUN_JOB} " "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " "-p {params.psf} -N {params.n_smp}" @@ -397,9 +338,11 @@ rule im_merge: cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + container: + SIF shell: "cd {params.run_dir} && " - "{EXEC_PIPELINE} python {CREATE_FINAL_CAT} " + "{_ENV_PREFIX} python {CREATE_FINAL_CAT} " "-I -m final_cat_{wildcards.sim}.hdf5 -i .. " "-p cfis/final_cat.param -P {wildcards.sim} " "-o n_tiles_final.txt -v" @@ -418,8 +361,10 @@ rule im_extract: cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + container: + SIF shell: - "cd {params.run_dir} && {EXEC} python {EXTRACT_INFO}" + "cd {params.run_dir} && {_ENV_PREFIX} python {EXTRACT_INFO}" rule im_calibrate: @@ -436,20 +381,16 @@ rule im_calibrate: cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + container: + SIF shell: "cd {params.run_dir} && " - "{EXEC} python {CALIBRATE} -s calibrate" + "{_ENV_PREFIX} python {CALIBRATE} -s calibrate" -rule im_mbias: - """Multiplicative/additive shear bias from the calibrated grids. - - Produces the workflow's headline artifact, ``m_bias_results.yaml``. The - injected shear (``shear_amplitude`` and the branch map) comes from - ``manifest.yaml`` alone -- no literal amplitude here or in config.yaml. The - generated ``m_bias_config.yaml`` carries the manifest's ``branches`` and - ``pairs``, so the estimator's sim list and pairing are the campaign's, not a - hard-coded default. +rule im_mbias_config: + """Assemble ``m_bias_config.yaml`` for the m-bias step: the manifest's + shear/branch facts, this run's science knobs, and git/container provenance. """ input: manifest=MANIFEST, @@ -457,112 +398,39 @@ rule im_mbias: f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS ), output: - results=f"{GRIDS_BASE}/results/m_bias_results.yaml", - params: cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", + params: grids_base=GRIDS_BASE, num=NUM, cat_name=f"shape_catalog_cut_{SHAPE}.fits", sif=SIF, - sif_pipeline=SIF_PIPELINE, shapepipe_repo=SHAPEPIPE_REPO, sp_validation_repo=SPV_REPO, + results_dir=f"{GRIDS_BASE}/results", + results=f"{GRIDS_BASE}/results/m_bias_results.yaml", # Science knobs, read bare from the run config (no default here). match_radius_deg=IMSIM["match_radius_deg"], w_cols=IMSIM["w_cols"], n_bootstrap=IMSIM["n_bootstrap"], pair_match=IMSIM["pair_match"], bootstrap_seed=IMSIM["bootstrap_seed"], - run: - import hashlib - import re - import subprocess - - import yaml - - with open(input.manifest) as fh: - manifest = yaml.safe_load(fh) - - def _git(repo, *args): - """Read a git fact from ``repo``; ``None`` if it is not a checkout.""" - try: - return subprocess.run( - ["git", "-C", repo, *args], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - def _sif_revision(sif_path): - """GHCR revision baked into the SIF's OCI labels. - - A plain-text scan of the image file (login-safe: no exec, no - container start), reading org.opencontainers.image.revision -- the - source commit GHCR built the image from. ``None`` if absent. - """ - try: - with open(sif_path, "rb") as fh: - blob = fh.read() - except OSError: - return None - m = re.search( - rb'org\.opencontainers\.image\.revision"?[:=]"?([0-9a-f]{7,40})', - blob, - ) - return m.group(1).decode() if m else None - - # Manifest hash: sha256 of the exact bytes im_manifest wrote, so the - # result records which injected-shear facts it was computed against. - with open(input.manifest, "rb") as fh: - manifest_sha256 = hashlib.sha256(fh.read()).hexdigest() - - provenance = { - "manifest_sha256": manifest_sha256, - "sp_validation": { - "branch": _git(params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), - }, - "shapepipe": { - "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), - }, - "containers": { - "sif": params.sif, - "ghcr_revision": _sif_revision(params.sif), - "sif_pipeline": params.sif_pipeline, - "ghcr_revision_pipeline": _sif_revision(params.sif_pipeline), - }, - } - - os.makedirs(os.path.dirname(output.results), exist_ok=True) - # Emit *every* key the estimator requires -- pair_match and - # bootstrap_seed included. Requiring a key without emitting it would - # be a KeyError at run time, so the generated config is the complete - # contract between rule and estimator. ``provenance`` rides along as a - # top-level block: the compute script copies it verbatim into the output - # results yaml, so a result file is self-describing (which manifest, - # which repo commits, which container built the number). - mbias_cfg = { - "grids_dir": params.grids_base, - "num": params.num, - "catalog_name": params.cat_name, - # Injected shear: from the manifest, the single source of truth. - "shear_amplitude": manifest["shear_amplitude"], - "branches": list(manifest["branches"]), - "pairs": manifest["pairs"], - "match_radius_deg": params.match_radius_deg, - "w_cols": list(params.w_cols), - "pair_match": params.pair_match, - "n_bootstrap": params.n_bootstrap, - "bootstrap_seed": params.bootstrap_seed, - "results_dir": os.path.dirname(output.results), - "output_path": output.results, - "provenance": provenance, - } - with open(params.cfg, "w") as fh: - yaml.safe_dump(mbias_cfg, fh) - shell( - "{EXEC} python {COMPUTE_M_BIAS} -c {params.cfg} -v" - ) + container: + SIF + script: + "../scripts/im_mbias_config.py" + + +rule im_mbias: + """Multiplicative/additive shear bias from the calibrated grids. + + Produces the workflow's headline artifact, ``m_bias_results.yaml``, running + the estimator against the config ``im_mbias_config`` assembled. + """ + input: + cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", + output: + results=f"{GRIDS_BASE}/results/m_bias_results.yaml", + container: + SIF + shell: + "{_ENV_PREFIX} python {COMPUTE_M_BIAS} -c {input.cfg} -v" diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2..861c3b05 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,46 +24,53 @@ rule xi: "../scripts/run_2pcf.py" -rule xi_highres: - """High-resolution xi for COSEBIS integration.""" - container: None - output: - txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - resources: - tasks=30, - cpus_per_task=12, - nodes=6, - mem_mb_per_cpu=2000, - runtime=2880, - slurm_extra="'--exclude=n17,n09,n36 --partition=pscomp'", - mpi="/softs/openmpi/5.0.5-slurm-CentOS8/bin/mpiexec", - shell: - "{resources.mpi} -n {resources.tasks} " - "apptainer exec " - "--bind /home,/n09data,/n17data,/n23data1,/softs " - "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " - "/n17data/cdaley/containers/containers " - "python /automnt/n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/run_2pcf_highres.py" - - -rule run_cosmo_val: - """Full CosmoVal diagnostic suite.""" - output: - sentinel=str(COSMO_VAL / "run_cosmo_val.done"), - threads: 24 - resources: - mem_mb=60000, - disk_mb=20000, - runtime=360, - shell: - """ - export PYTHONPATH="/home/cdaley/.local/lib/python3.12/site-packages:${{PYTHONPATH:-}}" - cd /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val \ - && python run_cosmo_val.py \ - && touch {output.sentinel} - """ +# PARKED: xi_highres (high-resolution xi for COSEBIS integration). Not runnable +# as written -- the shell invokes run_2pcf_highres.py bare, but the script has +# required --cat-config and --out arguments. Revive it with those supplied. +# +# The MPI reasoning below is hard-won and must survive the revival: +# +# Exception to the profile-driven container model: this is multi-node MPI, one +# `apptainer exec` per rank. Snakemake's own container wrapping puts the +# *whole* shell command -- `mpiexec` included -- inside a single container +# instance, so only rank 0's node would run inside it; the other ranks, +# spawned by SLURM/PMI on their own nodes, would land bare on the host. +# `container: None` plus an explicit `mpiexec -n N apptainer exec ...` +# per-rank is therefore required. +# Snakemake's slurm-jobstep plugin deliberately does NOT prepend `srun` to a +# job carrying an `mpi` resource, which is what lets the rule's own launcher +# run on the host, outside the container. +# Because this rule builds its own apptainer call, reaching the source-cache +# copy of the script relies on our `--bind /home` rather than on Snakemake's +# automatic mount -- and on a concrete image file, since `apptainer exec` +# takes no `docker://` URI. Take that path from `resolve_image()[0]` rather +# than naming a second image path that can drift. +# +# rule xi_highres: +# container: None +# params: +# image=resolve_image()[0], +# input: +# script=workflow.source_path("../scripts/run_2pcf_highres.py"), +# output: +# txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), +# xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), +# xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), +# resources: +# tasks=30, +# cpus_per_task=12, +# nodes=6, +# mem_mb_per_cpu=2000, +# runtime=2880, +# slurm_extra="'--exclude=n17,n09,n36 --partition=pscomp'", +# mpi="/softs/openmpi/5.0.5-slurm-CentOS8/bin/mpiexec", +# shell: +# "{resources.mpi} -n {resources.tasks} " +# "apptainer exec " +# "--bind /home,/n09data,/n17data,/n23data1,/softs " +# "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " +# "{params.image} " +# "python {input.script} --cat-config <...> --out <...>" rule rho_tau_stats: diff --git a/workflow/scripts/analyze_mask_power_spectrum.py b/workflow/scripts/analyze_mask_power_spectrum.py index 2e0e5b28..65c02132 100644 --- a/workflow/scripts/analyze_mask_power_spectrum.py +++ b/workflow/scripts/analyze_mask_power_spectrum.py @@ -72,8 +72,6 @@ def export_power_spectrum( def main(): """Process single mask power spectrum (Snakemake script entry point).""" - from snakemake.script import snakemake - mask_path = snakemake.input.mask output_path = str(snakemake.output.power_spectrum) diff --git a/workflow/scripts/cosmocov_process.py b/workflow/scripts/cosmocov_process.py new file mode 100644 index 00000000..6e6c1723 --- /dev/null +++ b/workflow/scripts/cosmocov_process.py @@ -0,0 +1,73 @@ +"""Assemble a raw CosmoCov block dump into an analysis-ready covariance matrix. + +CosmoCov writes one row per (i, j) element with the Gaussian term in column 8 +and the non-Gaussian term in column 9; this rebuilds the symmetric matrices, +checks positive-definiteness, and plots the correlation matrix. + +Run through Snakemake's ``script:`` directive, which injects ``snakemake`` as a +module global before this file executes. +""" + +import sys + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 + + +def get_cov(filename): + """Return (gaussian, non-gaussian, ndata) from a CosmoCov element list.""" + + data = np.loadtxt(filename) + ndata = int(np.max(data[:, 0])) + 1 + + cov_g = np.zeros((ndata, ndata)) + cov_ng = np.zeros((ndata, ndata)) + for i in range(data.shape[0]): + row, col = int(data[i, 0]), int(data[i, 1]) + cov_g[row, col] = cov_g[col, row] = data[i, 8] + cov_ng[row, col] = cov_ng[col, row] = data[i, 9] + + return cov_g, cov_ng, ndata + + +def plot_correlation(cov, ndata, plot_path): + """Save the correlation matrix with xi+/xi- block annotations.""" + + diag = np.sqrt(np.diag(cov)) + correlation = cov / np.outer(diag, diag) + + fig, ax = plt.subplots() + extent = (0, ndata, ndata, 0) + image = ax.imshow(correlation, cmap="seismic", vmin=-1, vmax=1, extent=extent) + + ax.axvline(x=ndata // 2, color="black", linewidth=1.0) + ax.axhline(y=ndata // 2, color="black", linewidth=1.0) + + fig.colorbar(image, orientation="vertical") + + ax.text(ndata // 4, ndata + 5, r"$\xi_+^{ij}(\theta)$", fontsize=12) + ax.text(3 * (ndata // 4), ndata + 5, r"$\xi_-^{ij}(\theta)$", fontsize=12) + ax.text(-9, ndata // 4, r"$\xi_+^{ij}(\theta)$", fontsize=12) + ax.text(-9, 3 * (ndata // 4), r"$\xi_-^{ij}(\theta)$", fontsize=12) + + fig.savefig(plot_path, dpi=300) + plt.close(fig) + + +cov_g, cov_ng, ndata = get_cov(snakemake.input[0]) # noqa: F821 +print(f"Dimension of cov: {ndata}x{ndata}") + +cov = cov_g + cov_ng + +eigenvalues = np.linalg.eigvalsh(cov) +print(f"min+max eigenvalues cov: {eigenvalues.min():e}, {eigenvalues.max():e}") +if eigenvalues.min() <= 0.0: + sys.exit("non-positive eigenvalue encountered! Covariance invalid!") + +np.savetxt(snakemake.output.matrix, cov) # noqa: F821 +np.savetxt(snakemake.output.gaussian, cov_g) # noqa: F821 +plot_correlation(cov, ndata, snakemake.output.plot) # noqa: F821 diff --git a/workflow/scripts/cv_additive_bias.py b/workflow/scripts/cv_additive_bias.py index 3fdec487..60df4c3d 100644 --- a/workflow/scripts/cv_additive_bias.py +++ b/workflow/scripts/cv_additive_bias.py @@ -10,7 +10,6 @@ import json from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 182cda99..c1ca8275 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -8,7 +8,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_footprints.py b/workflow/scripts/cv_footprints.py index 5ed89f07..e4a1af6a 100644 --- a/workflow/scripts/cv_footprints.py +++ b/workflow/scripts/cv_footprints.py @@ -6,7 +6,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_objectwise_leakage.py b/workflow/scripts/cv_objectwise_leakage.py index 8b6d5694..ae0f012a 100644 --- a/workflow/scripts/cv_objectwise_leakage.py +++ b/workflow/scripts/cv_objectwise_leakage.py @@ -8,7 +8,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_plot_2pcf.py b/workflow/scripts/cv_plot_2pcf.py index 9d5c8900..5251e7a5 100644 --- a/workflow/scripts/cv_plot_2pcf.py +++ b/workflow/scripts/cv_plot_2pcf.py @@ -7,7 +7,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_plot_rho_stats.py b/workflow/scripts/cv_plot_rho_stats.py index 95a37b09..c99bd646 100644 --- a/workflow/scripts/cv_plot_rho_stats.py +++ b/workflow/scripts/cv_plot_rho_stats.py @@ -6,7 +6,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_plot_tau_stats.py b/workflow/scripts/cv_plot_tau_stats.py index ed90e334..2a00b3a5 100644 --- a/workflow/scripts/cv_plot_tau_stats.py +++ b/workflow/scripts/cv_plot_tau_stats.py @@ -5,7 +5,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index cf04e8e8..7dfebc54 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -6,7 +6,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d15a763f..bcc46d84 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -9,7 +9,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_ratio_xi_sys_xi.py b/workflow/scripts/cv_ratio_xi_sys_xi.py index ba737d7e..79afd89a 100644 --- a/workflow/scripts/cv_ratio_xi_sys_xi.py +++ b/workflow/scripts/cv_ratio_xi_sys_xi.py @@ -8,7 +8,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_rho_tau_fits.py b/workflow/scripts/cv_rho_tau_fits.py index 39d2e371..64b24b67 100644 --- a/workflow/scripts/cv_rho_tau_fits.py +++ b/workflow/scripts/cv_rho_tau_fits.py @@ -9,7 +9,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 90df5999..85f0fa1d 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -17,7 +17,6 @@ import json from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_weights.py b/workflow/scripts/cv_weights.py index 3cb3f3ca..0316fe1e 100644 --- a/workflow/scripts/cv_weights.py +++ b/workflow/scripts/cv_weights.py @@ -5,7 +5,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/generate_glass_mock_rhotau_samples.py b/workflow/scripts/generate_glass_mock_rhotau_samples.py index 63d99095..87b7d06c 100644 --- a/workflow/scripts/generate_glass_mock_rhotau_samples.py +++ b/workflow/scripts/generate_glass_mock_rhotau_samples.py @@ -116,6 +116,36 @@ def generate_samples_for_mock(mock_id, cov_tau, theta, ref_tau_header, output_di return f"Generated tau samples for mock {mock_id:05d}" +def run(cov_tau_path, ref_tau_path, output_dir, mock_ids): + """Generate sampled tau statistics for every mock in ``mock_ids``.""" + print("Loading tau covariance...") + cov_tau = np.load(cov_tau_path) + print(f" cov_tau shape: {cov_tau.shape}") + + print("Loading reference FITS...") + ref_tau_data, ref_tau_header = load_reference_fits(ref_tau_path) + theta = ref_tau_data["theta"] + print( + f" theta range: {theta.min():.3f} - {theta.max():.3f} arcmin, nbins: {len(theta)}" + ) + + print(f"Generating tau samples for {len(mock_ids)} mocks...") + for mock_id in mock_ids: + msg = generate_samples_for_mock( + mock_id, cov_tau, theta, ref_tau_header, output_dir + ) + print(msg) + print("Done!") + + +def parse_mock_ids(spec): + """Parse a mock-ID spec, either a single ID or an inclusive ``lo-hi`` range.""" + if "-" in spec: + lo, hi = spec.split("-") + return list(range(int(lo), int(hi) + 1)) + return [int(spec)] + + def main(): parser = argparse.ArgumentParser( description="Generate zero-mean tau samples for GLASS mocks" @@ -146,37 +176,22 @@ def main(): ) args = parser.parse_args() - # Load tau covariance and reference - print("Loading tau covariance...") - cov_tau = np.load(args.cov_tau) - print(f" cov_tau shape: {cov_tau.shape}") - - print("Loading reference FITS...") - ref_tau_data, ref_tau_header = load_reference_fits(args.ref_tau) - theta = ref_tau_data["theta"] - print( - f" theta range: {theta.min():.3f} - {theta.max():.3f} arcmin, nbins: {len(theta)}" - ) + run(args.cov_tau, args.ref_tau, args.output_dir, parse_mock_ids(args.mock_ids)) - # Parse mock ID range - mock_ids = ( - list( - range( - int(args.mock_ids.split("-")[0]), int(args.mock_ids.split("-")[1]) + 1 - ) - ) - if "-" in args.mock_ids - else [int(args.mock_ids)] - ) - print(f"Generating tau samples for {len(mock_ids)} mocks...") - for mock_id in mock_ids: - msg = generate_samples_for_mock( - mock_id, cov_tau, theta, ref_tau_header, args.output_dir - ) - print(msg) - print("Done!") +# ── Entry point dispatch ───────────────────────────────────────────────────── +try: + snakemake # injected by snakemake's script: directive +except NameError: + snakemake = None -if __name__ == "__main__": +if snakemake is not None: + run( + snakemake.input.cov_tau, + snakemake.input.ref_tau, + snakemake.params.output_dir, + parse_mock_ids(snakemake.params.mock_id), + ) +elif __name__ == "__main__": main() diff --git a/workflow/scripts/im_mbias_config.py b/workflow/scripts/im_mbias_config.py new file mode 100644 index 00000000..cf6fdba2 --- /dev/null +++ b/workflow/scripts/im_mbias_config.py @@ -0,0 +1,105 @@ +"""Assemble ``m_bias_config.yaml`` for the image-sims m-bias estimator. + +Combines the manifest's injected-shear facts, this run's science knobs and +git/container provenance into the single config ``compute_m_bias_image_sims.py`` +reads. ``provenance`` rides along as a top-level block: the estimator copies it +verbatim into its results yaml, so a result file records which manifest, repo +commits and container produced the number. + +`snakemake` is injected as a module global by Snakemake's `script:` preamble +before this file runs (`from snakemake.script import snakemake` is +IDE-hint-only and raises ImportError if actually executed). +""" + +import hashlib +import os +import re +import subprocess + +import yaml + +params = snakemake.params # noqa: F821 +manifest_path = snakemake.input["manifest"] # noqa: F821 + + +def _git(repo, *args): + """Read a git fact from ``repo``; ``None`` if it is not a checkout.""" + try: + return subprocess.run( + ["git", "-C", repo, *args], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + +def _sif_revision(): + """``org.opencontainers.image.revision`` from the running image's OCI labels. + + Read the image actually mounted, which Apptainer names in + ``APPTAINER_CONTAINER``, rather than the configured ``sif`` -- a run may + override it, and a registry tag says nothing about which build was pulled. + Chunked plain-text scan: no exec, no container start, no 1.5 GB in memory. + """ + sif_path = os.environ.get("APPTAINER_CONTAINER") + if not sif_path: + return None + pattern = re.compile( + rb'org\.opencontainers\.image\.revision"?[:=]"?([0-9a-f]{7,40})' + ) + tail = b"" + try: + with open(sif_path, "rb") as fh: + while chunk := fh.read(8 << 20): + m = pattern.search(tail + chunk) + if m: + return m.group(1).decode() + tail = chunk[-128:] + except OSError: + return None + return None + + +with open(manifest_path) as fh: + manifest = yaml.safe_load(fh) +with open(manifest_path, "rb") as fh: + manifest_sha256 = hashlib.sha256(fh.read()).hexdigest() + +mbias_cfg = { + "grids_dir": params.grids_base, + "num": params.num, + "catalog_name": params.cat_name, + # Injected shear: from the manifest, the single source of truth. + "shear_amplitude": manifest["shear_amplitude"], + "branches": list(manifest["branches"]), + "pairs": manifest["pairs"], + "match_radius_deg": params.match_radius_deg, + "w_cols": list(params.w_cols), + "pair_match": params.pair_match, + "n_bootstrap": params.n_bootstrap, + "bootstrap_seed": params.bootstrap_seed, + "results_dir": params.results_dir, + "output_path": params.results, + "provenance": { + "manifest_sha256": manifest_sha256, + "sp_validation": { + "branch": _git( + params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD" + ), + "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), + }, + "shapepipe": { + "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), + "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), + }, + "container": { + "sif": params.sif, + "ghcr_revision": _sif_revision(), + }, + }, +} + +with open(snakemake.output["cfg"], "w") as fh: # noqa: F821 + yaml.safe_dump(mbias_cfg, fh) diff --git a/workflow/scripts/process_mask.py b/workflow/scripts/process_mask.py index c0368147..c9590368 100644 --- a/workflow/scripts/process_mask.py +++ b/workflow/scripts/process_mask.py @@ -138,8 +138,6 @@ def save_area_summary( def main(): """Main processing function.""" - # Snakemake script execution only (no interactive mode) - from snakemake.script import snakemake # Get parameters from Snakemake source_mask_path = snakemake.input.mask diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index ea2f35bc..fbf62a3c 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -28,8 +28,6 @@ "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.5.fits", "/home/cdaley/n17data/unions/pure_eb", ) -else: - from snakemake.script import snakemake params = snakemake.params # type: ignore From 5c03d047a096470c855a9f3a632f7c27421ed1bb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:05 +0200 Subject: [PATCH 08/83] Update actions/checkout action to v7 (#325) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/lint.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 689f4bcb..15dccde5 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install documentation dependencies run: uv pip install --no-cache-dir '.[docs]' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a0f36765..2c20ceaf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -80,7 +80,7 @@ jobs: steps: - name: Checkout (PR head on pull_request_target, else the pushed ref) - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false From 0fa080d1780b6a847a7f16511d009f7cf1a83f84 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:09 +0200 Subject: [PATCH 09/83] Update actions/github-script action to v9 (#326) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2c20ceaf..c9c369f5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -224,7 +224,7 @@ jobs: - name: Tell the author (PR comment) or record it (develop-push issue) if: steps.ruff.outputs.tool_error == 'false' continue-on-error: true - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: PASSED: ${{ steps.gate.outputs.passed }} AUTOFIXED: ${{ steps.gate.outputs.autofixed }} From b9955f3713e0fdb1a6de5825596b47125aef03c9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:14 +0200 Subject: [PATCH 10/83] Update actions/upload-artifact action to v7 (#327) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 15dccde5..691090fc 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -47,7 +47,7 @@ jobs: # Upload the rendered HTML so it can be downloaded from the run summary — # the only way to preview the docs on a PR, where the deploy is skipped. - name: Upload built docs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: docs-html path: docs/_build From 8293f9a792a5afb108a0d3d4e7701d6dd444cee2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:19 +0200 Subject: [PATCH 11/83] Update astral-sh/setup-uv action to v10 (#328) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c9c369f5..f1664d53 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -86,7 +86,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then # can we push an autofix commit back to it with the workflow token. From 11f92d1ce730a00e792151e9e3ce8e25ff4ae1da Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:23 +0200 Subject: [PATCH 12/83] Update docker/build-push-action action to v7 (#330) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 192e87a6..4add3347 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -35,7 +35,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and export to Docker - uses: docker/build-push-action@v6 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: load: true tags: ${{ steps.meta.outputs.tags }} @@ -52,7 +52,7 @@ jobs: run: docker run --rm ${{ steps.meta.outputs.tags }} python -m pytest src/sp_validation/tests -m "not slow" - name: Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: push: true tags: ${{ steps.meta.outputs.tags }} From f74f8e1a1b3626ac5f877d7e8ebe9e2c33df938d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:27 +0200 Subject: [PATCH 13/83] Update docker/login-action action to v4 (#331) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 4add3347..27bdeb1b 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} From 66173c5d3296565aa03af54c1da87cf52fd83d41 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:31 +0200 Subject: [PATCH 14/83] Update docker/metadata-action action to v6 (#332) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 27bdeb1b..9f6ce070 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -30,7 +30,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} From b622196b881ed62bc0d94b32886a51636ed4de2d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:17:36 +0200 Subject: [PATCH 15/83] Update docker/setup-buildx-action action to v4 (#333) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 9f6ce070..f4fa233d 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -26,7 +26,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - name: Extract metadata (tags, labels) for Docker id: meta From 3585b2e74790237e3e111c902b6110de9f007f0e Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Tue, 8 Sep 2026 18:34:46 +0200 Subject: [PATCH 16/83] ci: queue gh-pages deploys instead of racing (#335) Concurrent doc-deploy runs on back-to-back pushes to develop both hit peaceiris/actions-gh-pages at once, and the loser fails with "cannot lock ref 'refs/heads/gh-pages'" (seen in run 34250154443). Scope the job to a gh-pages- concurrency group; cancel-in-progress is fine since develop only ever has one active deploy target and the newest push should win over a stale one still building. Claude-Session: https://claude.ai/code/session_01Wk8SZkCRuKQ5xu5g38zHpx Co-authored-by: Claude Fable 5.1 --- .github/workflows/deploy-docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 691090fc..19f3edb5 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -23,6 +23,9 @@ jobs: permissions: contents: write packages: read + concurrency: + group: gh-pages-${{ github.ref }} + cancel-in-progress: true container: image: ghcr.io/cosmostat/sp_validation:develop From 5fbed605d291fd48418c2bfdc101551dba616635 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Wed, 9 Sep 2026 14:30:23 +0200 Subject: [PATCH 17/83] Fix fiducial pure-EB window resolution (#323) Resolve reporting bins from their edges rather than bin means. This keeps the 12--83 arcmin fiducial window at bins 9--15 and preserves the intended degrees of freedom. Co-authored-by: Claude Fable 5.1 --- .../scripts/config_space_pte_matrices.py | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/papers/bmodes/scripts/config_space_pte_matrices.py b/papers/bmodes/scripts/config_space_pte_matrices.py index 70044149..474b71a7 100644 --- a/papers/bmodes/scripts/config_space_pte_matrices.py +++ b/papers/bmodes/scripts/config_space_pte_matrices.py @@ -42,6 +42,14 @@ plt.style.use(PAPER_MPLSTYLE) +def resolve_fiducial_bin_window(edges, theta_min, theta_max): + """Return the first and last reporting bins inside a scale-cut window.""" + left, right = edges[:-1], edges[1:] + inside = (left >= theta_min * (1.0 - 1e-2)) & (right <= theta_max * (1.0 + 1e-2)) + bins = np.flatnonzero(inside) + return int(bins[0]), int(bins[-1]) + + def _path_matches_version(path, version): """Check if a file path matches a specific catalog version exactly. @@ -518,10 +526,13 @@ def create_3panel_composite( cosebis_fid_start = np.argmin(np.abs(theta_cosebis[:-1] - cosebis_fid[0])) cosebis_fid_stop = np.argmin(np.abs(theta_cosebis[1:] - cosebis_fid[1])) + 1 - xip_start = np.argmin(np.abs(theta_pure_eb - xip_fid[0])) - xip_stop = np.argmin(np.abs(theta_pure_eb - xip_fid[1])) - xim_start = np.argmin(np.abs(theta_pure_eb - xim_fid[0])) - xim_stop = np.argmin(np.abs(theta_pure_eb - xim_fid[1])) + reporting_edges = np.geomspace( + config["fiducial"]["min_sep"], + config["fiducial"]["max_sep"], + config["fiducial"]["nbins"] + 1, + ) + xip_start, xip_stop = resolve_fiducial_bin_window(reporting_edges, *xip_fid) + xim_start, xim_stop = resolve_fiducial_bin_window(reporting_edges, *xim_fid) # Create subplot axes ax_xip = fig.add_subplot(gs[0, 0]) @@ -673,10 +684,13 @@ def create_9panel_composite( cosebis_fid_start = np.argmin(np.abs(theta_cosebis[:-1] - cosebis_fid[0])) cosebis_fid_stop = np.argmin(np.abs(theta_cosebis[1:] - cosebis_fid[1])) + 1 - xip_start = np.argmin(np.abs(theta_pure_eb - xip_fid[0])) - xip_stop = np.argmin(np.abs(theta_pure_eb - xip_fid[1])) - xim_start = np.argmin(np.abs(theta_pure_eb - xim_fid[0])) - xim_stop = np.argmin(np.abs(theta_pure_eb - xim_fid[1])) + reporting_edges = np.geomspace( + config["fiducial"]["min_sep"], + config["fiducial"]["max_sep"], + config["fiducial"]["nbins"] + 1, + ) + xip_start, xip_stop = resolve_fiducial_bin_window(reporting_edges, *xip_fid) + xim_start, xim_stop = resolve_fiducial_bin_window(reporting_edges, *xim_fid) # Create subplot axes for this row ax_xip = fig.add_subplot(gs[row_idx, 0]) @@ -900,12 +914,18 @@ def main( config, fiducial_overrides, ) - theta_pe = matrices["theta_pure_eb"] theta_co = matrices["theta_cosebis"] - xip_start = np.argmin(np.abs(theta_pe - xip_fid[0])) - xip_stop = np.argmin(np.abs(theta_pe - xip_fid[1])) - xim_start = np.argmin(np.abs(theta_pe - xim_fid[0])) - xim_stop = np.argmin(np.abs(theta_pe - xim_fid[1])) + reporting_edges = np.geomspace( + config["fiducial"]["min_sep"], + config["fiducial"]["max_sep"], + config["fiducial"]["nbins"] + 1, + ) + xip_start, xip_stop = resolve_fiducial_bin_window( + reporting_edges, *xip_fid + ) + xim_start, xim_stop = resolve_fiducial_bin_window( + reporting_edges, *xim_fid + ) cos_start = np.argmin(np.abs(theta_co[:-1] - cosebis_fid[0])) cos_stop = np.argmin(np.abs(theta_co[1:] - cosebis_fid[1])) + 1 From 6de9a92bb4b92071cb7b31dfc9bf74063a713b11 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Wed, 9 Sep 2026 14:31:08 +0200 Subject: [PATCH 18/83] Add TeX distribution to container image (#338) Co-authored-by: Claude Fable 5.1 --- Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 79fdf0d8..560df4b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,14 @@ RUN apt-get update -y --quiet --fix-missing && \ liblapack-dev \ libgsl-dev \ libcfitsio-dev \ - libfftw3-dev + libfftw3-dev \ + texlive-latex-base \ + texlive-latex-recommended \ + texlive-fonts-recommended \ + dvipng \ + ghostscript \ + cm-super && \ + rm -rf /var/lib/apt/lists/* # The base shapepipe image provides a uv-managed venv at /app/.venv (exported as # VIRTUAL_ENV); install sp_validation's deps into that same venv rather than From 88fa75bbd6b22dcb1f54a8ed66d568deb71ad914 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 16:04:08 +0200 Subject: [PATCH 19/83] Container: TinyTeX (pinned TeX Live 2025) instead of apt texlive The texlive-latex-base/-recommended/-fonts-recommended + cm-super set added in #338 costs ~1.3 GB and still lacks type1cm.sty, so matplotlib's usetex path stays broken. Replace it with TinyTeX (~270 MB measured), pinned to TeX Live 2025 on both halves: bundle v2026.02, and tlmgr pointed at that year's frozen tlnet-final historic mirror. Packages are an explicit tlmgr list covering matplotlib's usetex preamble. Co-Authored-By: Claude Fable 5.1 --- Dockerfile | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 560df4b2..2124d2fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,14 +20,45 @@ RUN apt-get update -y --quiet --fix-missing && \ libgsl-dev \ libcfitsio-dev \ libfftw3-dev \ - texlive-latex-base \ - texlive-latex-recommended \ - texlive-fonts-recommended \ - dvipng \ - ghostscript \ - cm-super && \ + perl \ + curl \ + ghostscript && \ rm -rf /var/lib/apt/lists/* +# TeX, for matplotlib's usetex path (paper.mplstyle) and any LaTeX output. +# TinyTeX rather than apt's texlive-*: those metapackages cost ~1.3 GB and +# still omit type1cm.sty, which matplotlib's usetex preamble loads. TinyTeX is +# a minimal TeX Live with tlmgr, so the package set below is explicit and +# adding one is a one-line change. +# +# Both halves are pinned to TeX Live 2025: the TinyTeX bundle v2026.02 is the +# last release built on it, and tlmgr points at that year's frozen tlnet-final +# historic mirror, so this layer resolves the same way forever. Update once a +# year by bumping TINYTEX_VERSION and the mirror year together. +ENV TEXLIVE_YEAR=2025 \ + TINYTEX_VERSION=2026.02 \ + TINYTEX_DIR=/opt \ + PATH=/opt/.TinyTeX/bin/x86_64-linux:$PATH +RUN set -eux; \ + curl -fsSL https://yihui.org/tinytex/install-bin-unix.sh | sh; \ + tlmgr option sys_bin /usr/local/bin; \ + tlmgr option repository \ + "https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${TEXLIVE_YEAR}/tlnet-final/"; \ + tlmgr option docfiles 0; \ + tlmgr option srcfiles 0; \ + tlmgr install \ + type1cm \ + cm-super \ + dvipng \ + underscore \ + ulem \ + amsmath \ + amsfonts \ + geometry \ + xcolor; \ + tlmgr path add; \ + latex --version >/dev/null; dvipng --version >/dev/null + # The base shapepipe image provides a uv-managed venv at /app/.venv (exported as # VIRTUAL_ENV); install sp_validation's deps into that same venv rather than # spawning a second one under /sp_validation. From e9aa8ac707a1973669d2bf9b5073bb2ccad07d3c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 9 Sep 2026 23:16:42 +0200 Subject: [PATCH 20/83] Container: trim the TinyTeX comment Co-Authored-By: Claude Fable 5.1 --- Dockerfile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2124d2fa..844f7e23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,16 +25,7 @@ RUN apt-get update -y --quiet --fix-missing && \ ghostscript && \ rm -rf /var/lib/apt/lists/* -# TeX, for matplotlib's usetex path (paper.mplstyle) and any LaTeX output. -# TinyTeX rather than apt's texlive-*: those metapackages cost ~1.3 GB and -# still omit type1cm.sty, which matplotlib's usetex preamble loads. TinyTeX is -# a minimal TeX Live with tlmgr, so the package set below is explicit and -# adding one is a one-line change. -# -# Both halves are pinned to TeX Live 2025: the TinyTeX bundle v2026.02 is the -# last release built on it, and tlmgr points at that year's frozen tlnet-final -# historic mirror, so this layer resolves the same way forever. Update once a -# year by bumping TINYTEX_VERSION and the mirror year together. +# TinyTeX pinned to a TeX Live year (frozen tlnet-final mirror); bump both once a year. ENV TEXLIVE_YEAR=2025 \ TINYTEX_VERSION=2026.02 \ TINYTEX_DIR=/opt \ From dc5886c5d65895118246a8c750666a863f121860 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:28:20 +0000 Subject: [PATCH 21/83] Update all non-major dependencies (#345) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- pyproject.toml | 4 +-- uv.lock | 67 ++++++++++++++++++++++++-------------- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f1664d53..8a0f226e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -86,7 +86,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then # can we push an autofix commit back to it with the workflow token. diff --git a/pyproject.toml b/pyproject.toml index 071945b1..5c45cfbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ dependencies = [ # be converted to Python scalars`, failing all get_cosmo-backed tests. camb # 1.6.6 (latest on PyPI) and current master are both unfixed. Lift this cap # when camb ships > 1.6.6 with the scipy-1.18 fix. - "scipy<1.18", + "scipy<1.19", "seaborn", # Intentionally tracks the mutable `develop` branch (NOT SHA-pinned): # shear_psf_leakage is actively co-developed CosmoStat code we want to stay @@ -148,7 +148,7 @@ docs = [ # lives in the separate ``glass.ext.camb`` package (absent from glass core # >=2024.2), so it is pinned explicitly. glass = [ - "glass==2025.1", + "glass==2025.3", "glass.ext.camb==2023.6", "cosmology==2022.10.9", # fitsio: make_unions_glass_sim.py writes the mock catalogue as FITS. diff --git a/uv.lock b/uv.lock index 4f635d14..eb69507c 100644 --- a/uv.lock +++ b/uv.lock @@ -112,6 +112,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/16/1a8fd2b19544b84575cf84ef7aa3ad4c173b756d5f087c91f85d1b295777/array_api_compat-1.15.0-py3-none-any.whl", hash = "sha256:7b1b9c53269061403fd5f45a8de349f16e7887653328bfa0c5f2d45299ff0a8e", size = 79113, upload-time = "2026-06-07T20:53:23.621Z" }, ] +[[package]] +name = "array-api-extra" +version = "0.11.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "array-api-compat" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/63/eb18bc7d039c954da3cfb11293c7e461b03eb9682bb0b22733be43f75501/array_api_extra-0.11.4.tar.gz", hash = "sha256:66236f72aaa4cdf97a3c5581696e3a9c89513714612a9bc6010e19fad1a22012", size = 112368, upload-time = "2026-09-07T18:50:34.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl", hash = "sha256:b2d5bd31b3072249fd3f90a2b1f7b398c78e3ccb7575e04e9e3348ae894da60f", size = 120196, upload-time = "2026-09-07T18:50:32.88Z" }, +] + [[package]] name = "arrow" version = "1.4.0" @@ -563,7 +575,7 @@ wheels = [ [[package]] name = "cosmosis" -version = "3.25.2" +version = "3.26" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dulwich" }, @@ -581,7 +593,7 @@ dependencies = [ { name = "threadpoolctl" }, { name = "zeus-mcmc" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/63/83898440e5486294cb17a7ddf01896af66dd153d7f3e77897ed120abd055/cosmosis-3.25.2.tar.gz", hash = "sha256:4a8333395b600a5e8339c637b947a982bac4c88a7b242e4b6de8274be03004a8", size = 416233, upload-time = "2026-03-09T12:01:08.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/46/27/923408c09bc803b349a216974481fc075cb56f54b1ea949efe1233a294f1/cosmosis-3.26.tar.gz", hash = "sha256:bbeed481411d2bfa83d0b14bcc3d16a0b80dcf1c1f099e5066d10e4711c36508", size = 421446, upload-time = "2026-09-03T09:22:44.179Z" } [[package]] name = "coverage" @@ -1084,18 +1096,18 @@ wheels = [ [[package]] name = "glass" -version = "2025.1" +version = "2025.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cosmology" }, + { name = "array-api-compat" }, + { name = "array-api-extra" }, { name = "healpix" }, { name = "healpy" }, - { name = "numpy" }, { name = "transformcl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a5/d6/26cd58e75834671259f26b9287a584d726e6023da35ddf7ca3b7f2c393fb/glass-2025.1.tar.gz", hash = "sha256:7b1aa2394e16010f7f1b4243f49e7e12d7a4dd28fcbf3e3f7cf25ce4905a8615", size = 48533, upload-time = "2025-02-21T18:43:48.046Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/18/6d1387d74f597aacde573ffa9e7d28faa23c74414f69f56bffe9d3ccffaf/glass-2025.3.tar.gz", hash = "sha256:1f55bd5e85dc4eb3964bc6cf9a61e8da0cba4b3ecd60531d58384a27debcda3d", size = 274314, upload-time = "2025-12-12T16:38:18.096Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/e7/3e89bc5147a3e84245c783098afffbbacbcdf5678ba1aacc2c9c035434da/glass-2025.1-py3-none-any.whl", hash = "sha256:d7919a7d19e05ab8da4e52dbfbcfb5bd1ed2b72e0cc3afde39ec94ced22883c5", size = 47185, upload-time = "2025-02-21T18:43:46.649Z" }, + { url = "https://files.pythonhosted.org/packages/6e/99/25a79eb65032d54868b4fc85fd105741858f493c391e9663954c15398e80/glass-2025.3-py3-none-any.whl", hash = "sha256:1cd0cd21aa25f76917235812164a9aa666032222ffb7bacfae766cd12f62b26b", size = 58522, upload-time = "2025-12-12T16:38:16.557Z" }, ] [[package]] @@ -1598,11 +1610,14 @@ wheels = [ [[package]] name = "joblib" -version = "1.5.3" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +dependencies = [ + { name = "cloudpickle" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/1d/537ab090f302b838943a1b56497dd53059b9a9b46a074936470173a2e207/joblib-1.6.0.tar.gz", hash = "sha256:2ccc96785b12046c08fd6d55839c12857831b54a3c1673ffadd2f04bfc4eda03", size = 327903, upload-time = "2026-08-31T09:39:04.122Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/84099323c2ec4be98d935f63c033ac4151ee83836ca1050ede3b3aadf155/joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba", size = 306115, upload-time = "2026-08-31T09:39:02.298Z" }, ] [[package]] @@ -1893,7 +1908,7 @@ wheels = [ [[package]] name = "jupytext" -version = "1.19.4" +version = "1.19.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, @@ -1902,9 +1917,9 @@ dependencies = [ { name = "packaging" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3b/52/e014296ac8f40ca783aeb73dae52e65edbb0eaae0dcdc1ea41bfaa8aebf7/jupytext-1.19.4.tar.gz", hash = "sha256:739bcd4bc12aa4fe298a38017cdb5ae27b08a6ba3a5470728d2fe9e04b155db1", size = 4581977, upload-time = "2026-06-21T21:48:58.32Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/473f8ebb101553fb2ea6ab1d34324d6677844c968947ac050c759d539f2c/jupytext-1.19.5.tar.gz", hash = "sha256:605026446d605aa54fd7f7fc69df6ae51c7a46053d4cebf05afdc64d66de3df0", size = 4600916, upload-time = "2026-07-21T22:00:29.198Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/e9/e2ae007456069dbe01865c69a4203a7ada6f7e337b78fc2f12e51bd3fae7/jupytext-1.19.4-py3-none-any.whl", hash = "sha256:032d4ef4bd2e96addcac780b9b1d6b5a266ca39beceaaca95bfb4f06e0b77029", size = 170889, upload-time = "2026-06-21T21:48:56.352Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c6/76ee9dacedcd8c67d8fa53dd975613733bdd28242a4c41518ff1c8aeaa64/jupytext-1.19.5-py3-none-any.whl", hash = "sha256:af22351202171116b986fe1f4f9233a13ca4a85d5eec57a87900ed48521dbae4", size = 229246, upload-time = "2026-07-21T22:00:27.218Z" }, ] [[package]] @@ -2597,14 +2612,14 @@ wheels = [ [[package]] name = "numpydoc" -version = "1.10.0" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sphinx" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/2f/5e9bf8ed0a93587b8089242b4926dfd04d4b3b0b65f04345d219a1225f03/numpydoc-1.11.0.tar.gz", hash = "sha256:26db0adaac8087e98da05c86b11198defd962aef7c21f4dffd2703c8c6eb4187", size = 101575, upload-time = "2026-09-15T16:31:08.905Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/5e/3a6a3e90f35cea3853c45e5d5fb9b7192ce4384616f932cf7591298ab6e1/numpydoc-1.10.0-py3-none-any.whl", hash = "sha256:3149da9874af890bcc2a82ef7aae5484e5aa81cb2778f08e3c307ba6d963721b", size = 69255, upload-time = "2025-12-02T16:39:11.561Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e5/f5c38d4899cd8edd5b522071c5658202077d113c3225484d779cf257970e/numpydoc-1.11.0-py3-none-any.whl", hash = "sha256:493ad2ce0d8275ca5da484ec37dbdbbc1334c3a6968dffd10ec729b51753e15e", size = 71464, upload-time = "2026-09-15T16:31:07.521Z" }, ] [[package]] @@ -3907,7 +3922,7 @@ requires-dist = [ { name = "fast-pt", marker = "extra == 'workflow'", specifier = ">=3.2,<4" }, { name = "fitsio", marker = "extra == 'glass'" }, { name = "getdist", git = "https://github.com/benabed/getdist.git?rev=113cd22a9a0d013b6f72fe734be81f260f3d3be5" }, - { name = "glass", marker = "extra == 'glass'", specifier = "==2025.1" }, + { name = "glass", marker = "extra == 'glass'", specifier = "==2025.3" }, { name = "glass-ext-camb", marker = "extra == 'glass'", specifier = "==2023.6" }, { name = "h5py" }, { name = "healpy" }, @@ -3937,7 +3952,7 @@ requires-dist = [ { name = "reproject" }, { name = "ruff", marker = "extra == 'test'" }, { name = "sacc", specifier = ">=0.12" }, - { name = "scipy", specifier = "<1.18" }, + { name = "scipy", specifier = "<1.19" }, { name = "seaborn" }, { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, { name = "skyproj" }, @@ -4346,7 +4361,7 @@ wheels = [ [[package]] name = "treecorr" -version = "5.1.3" +version = "5.1.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lsstdesc-coord" }, @@ -4355,12 +4370,16 @@ dependencies = [ { name = "pyyaml" }, { name = "setuptools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/95/ff399cb37b9e3d7c758f8cf0d5757f06a38093f223f2a65d7cc5b50313d5/treecorr-5.1.3.tar.gz", hash = "sha256:d921ccace064325a8ebb93f371c624f13fcc9559373e3b37cc216cb94ae81ed2", size = 1146147, upload-time = "2026-03-18T18:38:03.626Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/09/c9228fec3dc78910718f2a68306d7b0583b2dae93d6a8ce665ec0e19d9c7/treecorr-5.1.4.tar.gz", hash = "sha256:1514cbbc3ce843b875ea4968d69508554ea240f6738f6b544951969260d58a3f", size = 1149471, upload-time = "2026-09-01T02:33:56.976Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/97/74/1cc590e4fe1afb69ec2f10a67ee8e0a92ecafd36512052755362ff302d9c/treecorr-5.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596b6f5eea3115a9e211dc83ec6465887983f41b24e484e08fc47170c6cf4e26", size = 20941767, upload-time = "2026-03-18T18:37:37.874Z" }, - { url = "https://files.pythonhosted.org/packages/6f/bb/5ba5aa1021fce0cf459fa264cb0d298dfd095ddb48d63fb6589b90fdefac/treecorr-5.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:660dacd47a9af18790de279bcb6e7f375369096ad77e7cc64bf21a9b78f6c430", size = 19785388, upload-time = "2026-03-18T18:37:40.573Z" }, - { url = "https://files.pythonhosted.org/packages/be/92/be01211263a4fa488e768c9288c6f9b0f4402870423a88de4d60031b3384/treecorr-5.1.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73c938c6c31e1bad4d5338ad4519d1d78eedf3b7ef579b1e3a19d983e220ed93", size = 20940568, upload-time = "2026-03-18T18:37:44.966Z" }, - { url = "https://files.pythonhosted.org/packages/dc/a3/69e5a16aa33600ba22d28a47c6d8de3b3118292e79f9ae77dc69a3df7c77/treecorr-5.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc17b1f623bae6769148f57cac8c3a295e26e5d6215698117bd97e14c8bb0d6b", size = 19781991, upload-time = "2026-03-18T18:37:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/9f52a14c7e48d30875a2e988196554c74affc00ddf3571efb8f1fc9e901b/treecorr-5.1.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f12d28573a714f105c545921fd4eae8761f69c59b8eb2bc52080a5e3ed07b0d", size = 21593422, upload-time = "2026-09-01T02:33:04.719Z" }, + { url = "https://files.pythonhosted.org/packages/76/cf/b3ae27f1ef857491268aff4317999995ce939171c545a28363fbc058a961/treecorr-5.1.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f037dd80385f24158a6e1e1dae963acc904052a346d2b2006da47eff1ad51d", size = 21158309, upload-time = "2026-09-01T02:33:08.284Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c1/c577503c995d795e31e0a445ad2760d3fa42fb3f314dd3cd1a827fd211b0/treecorr-5.1.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a377144861b53d10c1d1cc9ff6dfb9878d44f74ef74943c8990b329925169ac3", size = 21593786, upload-time = "2026-09-01T02:33:14.165Z" }, + { url = "https://files.pythonhosted.org/packages/cd/96/79e4931447b0b6f42d712bfd5c1b0c171444b1e2e2d99db1528cd8094863/treecorr-5.1.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da932b75e481f18a8a55ac7f761c402d77472e6856744451124a93ad7f6f7273", size = 21153410, upload-time = "2026-09-01T02:33:17.932Z" }, + { url = "https://files.pythonhosted.org/packages/20/88/9f654adaf4baf2005300ed30390c526bd35c098bea1c1ac08cc04d17694e/treecorr-5.1.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c29c61636570815566aa0ab04738ffd98dc5029f06aff7fb1b98aaf4e44d12cc", size = 21585667, upload-time = "2026-09-01T02:33:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/79/60/f9ea6d17600f8a023b57b006c687bea479f5800aaeb2656429b82a625e12/treecorr-5.1.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c36fb089c8db8b77a230c90007007f1686661748c438bc5ac68640254bde478e", size = 21141746, upload-time = "2026-09-01T02:33:26.924Z" }, + { url = "https://files.pythonhosted.org/packages/37/bd/97c7aab98013f12cf1ae61a72dac4bff7faaf4f2bcd0a2aa5dcdd99ba748/treecorr-5.1.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:67f61ea796a2e3a487638e73abca9183187079f6e82110745e6cb5f4d8a086aa", size = 21583083, upload-time = "2026-09-01T02:33:32.311Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2f/4378607f9c7a922d78391b5b3e457e2600ff152b143e445c05078e32d428/treecorr-5.1.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:be9f97d5e73872a07c8cb5a4a132cab76c71f5ae357410fca2795ba81332c680", size = 21156480, upload-time = "2026-09-01T02:33:36.367Z" }, ] [[package]] From 607dff2acb4020511ce8b2b0b845e1a8f8d0cd00 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:42:12 +0000 Subject: [PATCH 22/83] Pin peaceiris/actions-gh-pages action to 84c30a8 --- .github/workflows/deploy-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 19f3edb5..3efce06b 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -57,7 +57,7 @@ jobs: - name: Deploy to GitHub Pages if: github.event_name == 'push' && github.ref == 'refs/heads/develop' - uses: peaceiris/actions-gh-pages@v4 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: docs/_build From 120e86c70fda2fa4c508dd5009ef1d383d5def9e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 05:25:55 +0000 Subject: [PATCH 23/83] Update docker/build-push-action digest to c3c9e26 (#347) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index f4fa233d..3c8ceaa0 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -35,7 +35,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and export to Docker - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: load: true tags: ${{ steps.meta.outputs.tags }} @@ -52,7 +52,7 @@ jobs: run: docker run --rm ${{ steps.meta.outputs.tags }} python -m pytest src/sp_validation/tests -m "not slow" - name: Push - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: push: true tags: ${{ steps.meta.outputs.tags }} From 2c561dec19b38c095f142a28e9e1a29da9bd12cd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:03:03 +0000 Subject: [PATCH 24/83] Update docker/setup-buildx-action digest to f87e599 (#348) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/deploy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 3c8ceaa0..f65ab5f7 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -26,7 +26,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Extract metadata (tags, labels) for Docker id: meta From efb32b2082f39ae714ddcdda907069cbc92fb591 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 14:41:17 +0200 Subject: [PATCH 25/83] chore: remove dead scripts and fix stale doc paths Deletes code with no live caller: - workflow/scripts/run_2pcf_highres.py and the parked, unrunnable xi_highres block in workflow/rules/twopoint.smk that was its only reference; the mpi4py comment in pyproject.toml no longer names it. - papers/bmodes/scripts/run_cov_sweep.sh, which calls workflow/scripts/run_cosmocov_chain.sh, absent from develop. Fixes docs that point at paths or tools that do not exist: - CLAUDE.md, CONTRIBUTING.md: single-test example uses test_cosmo_val.py. - CLAUDE.md: cosmo_inference runs through Snakemake (inference_fiducial), not a pipeline.sh driver. - papers/{bmodes,cosmo_val}/Snakefile: read cat_config through code/ rather than the deprecated pure_eb symlink. - ecut_spec.md, update_survey_stats.py: paths under papers/bmodes/. - README badge and installation docs: Python 3.12 floor, matching requires-python. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR --- CLAUDE.md | 14 +- CONTRIBUTING.md | 2 +- README.md | 2 +- docs/source/installation.rst | 2 +- papers/bmodes/Snakefile | 2 +- papers/bmodes/config/ecut_spec.md | 10 +- papers/bmodes/scripts/run_cov_sweep.sh | 68 ---- papers/bmodes/scripts/update_survey_stats.py | 2 +- papers/cosmo_val/Snakefile | 2 +- pyproject.toml | 4 +- workflow/rules/twopoint.smk | 49 --- workflow/scripts/run_2pcf_highres.py | 394 ------------------- 12 files changed, 21 insertions(+), 530 deletions(-) delete mode 100755 papers/bmodes/scripts/run_cov_sweep.sh delete mode 100644 workflow/scripts/run_2pcf_highres.py diff --git a/CLAUDE.md b/CLAUDE.md index c35c20be..b4447758 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ Tests live in `src/sp_validation/tests/` and import the full scientific stack, so run them inside the container. - Run all tests: `pytest` (collects from `src/sp_validation/tests`; coverage on by default) - Skip the slow tests: `pytest -m "not slow"` -- Run a single test: `pytest src/sp_validation/tests/test_cosmology.py::test_function_name` +- Run a single test: `pytest src/sp_validation/tests/test_cosmo_val.py::test_function_name` CI runs this same suite inside the freshly-built image before publishing it (see `.github/workflows/deploy-image.yml`). @@ -57,11 +57,13 @@ is the container (full scientific stack pre-built). For a local dev environment: - **Healpy/HealSparse**: Sky map handling ### Cosmology Inference Pipeline (`cosmo_inference/`) -Run via `./pipeline.sh` with flags: -- `--pcf`: Calculate 2-point correlation functions -- `--covmat`: Calculate covariance matrix with CosmoCov -- `--inference`: Run CosmoSIS inference -- `--mcmc_process`: Analyze MCMC chains +Orchestrated through Snakemake, not a standalone driver; see +`cosmo_inference/README.md`. From the repository root: + +```bash +snakemake --profile workflow/profiles/candide -s workflow/Snakefile \ + inference_fiducial --configfile +``` ### Configuration Main configuration in `scripts/calibration/params.py` with parameters: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e385a386..0406741b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ toolchain (`autoconf`, `automake`, `libtool`, `pkg-config`) available. ```bash pytest # full suite pytest -m "not slow" # skip the slow tests -pytest src/sp_validation/tests/test_cosmology.py::test_name # a single test +pytest src/sp_validation/tests/test_cosmo_val.py::test_name # a single test ``` Tests live in `src/sp_validation/tests/`. The default options (configured in diff --git a/README.md b/README.md index 455ddd91..1d269a0c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Validation of weak-lensing catalogues (galaxy and star shapes and other paramete [![docs](https://img.shields.io/badge/docs-sphinx-blue)](https://cosmostat.github.io/sp_validation/) [![CI](https://github.com/CosmoStat/sp_validation/actions/workflows/deploy-image.yml/badge.svg)](https://github.com/CosmoStat/sp_validation/actions/workflows/deploy-image.yml) [![container](https://img.shields.io/badge/container-ghcr.io-2496ED?logo=docker&logoColor=white)](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation) -[![python](https://img.shields.io/badge/python-3.11%2B-blue?logo=python&logoColor=white)](https://www.python.org/downloads/) +[![python](https://img.shields.io/badge/python-3.12%2B-blue?logo=python&logoColor=white)](https://www.python.org/downloads/) [![license](https://img.shields.io/badge/license-MIT-blue)](https://github.com/CosmoStat/sp_validation/blob/develop/LICENCE.txt) [![code style: ruff](https://img.shields.io/badge/code%20style-ruff-261230?logo=ruff&logoColor=white)](https://github.com/astral-sh/ruff) [![contribute](https://img.shields.io/badge/contribute-read-lightgrey)](https://github.com/CosmoStat/sp_validation/blob/develop/CONTRIBUTING.md) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 56de599f..22ea84cb 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -82,6 +82,6 @@ For a narrower install, ``.[test]`` adds only the test extras and ``.[docs]`` on .. note:: - ``sp_validation`` requires Python 3.11 or newer and pulls in a large scientific stack: ``treecorr``, ``pyccl``, ``healpy``, ``pymaster``, and others. + ``sp_validation`` requires Python 3.12 or newer and pulls in a large scientific stack: ``treecorr``, ``pyccl``, ``healpy``, ``pymaster``, and others. A bare development install builds these from source, which is slow and platform-sensitive. For most users the container is the more reliable path. diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index c58663d6..c9069c79 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -2,7 +2,7 @@ # compute workflow at ../../workflow/. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" +configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/papers/bmodes/config/ecut_spec.md b/papers/bmodes/config/ecut_spec.md index 49da3a92..53041659 100644 --- a/papers/bmodes/config/ecut_spec.md +++ b/papers/bmodes/config/ecut_spec.md @@ -32,7 +32,7 @@ DES-Y3 used e < 0.8 to remove stars. ### How versions flow through the pipeline -Everything is driven by `config["versions"]` in `workflow/config/config.yaml`. Adding a +Everything is driven by `config["versions"]` in `papers/bmodes/config/config.yaml`. Adding a version there (plus its `cat_config.yaml` entry) makes it flow through all existing rules: `xi`, `covariance`, `pure_eb_data_vector`, `cosebis_data_vector`, `cl_data_vector`. The 2PCF and covariance are independent and can run in parallel. @@ -43,7 +43,7 @@ Key resolution functions in `workflow/Snakefile`: - `resolve_covariance_version()` — identity function (each version gets its own covariance) - Wildcard constraint: `version=r"SP_v[\d.]+(_w_iv)?(_leak_corr)?"` — needs `_ecut\d+` -Version comparison rules in `workflow/rules/claims.smk` (lines 131, 271, 386) use +Version comparison rules in `papers/bmodes/rules/claims.smk` (lines 131, 271, 386) use `VERSIONS_LEAK_CORR` for inputs and derive version lists from config in the scripts. These should be parameterized to accept a version list via `snakemake.params`, so the same rules serve both paper and ecut comparisons. @@ -71,10 +71,10 @@ uncorrected columns can't be consistently filtered to guarantee the same rows. | What | Where | |------|-------| -| Workflow config | `workflow/config/config.yaml` (search `ecut`) | -| Catalog config | `code/sp_validation/cosmo_val/cat_config.yaml` (search `ecut07`) | +| Workflow config | `papers/bmodes/config/config.yaml` (search `ecut`) | +| Catalog config | `cosmo_val/cat_config.yaml` (search `ecut07`) | | Pipeline orchestration | `workflow/Snakefile` (wildcard constraints, version resolution functions) | -| Version comparison rules | `workflow/rules/claims.smk` lines 131, 271, 386 | +| Version comparison rules | `papers/bmodes/rules/claims.smk` lines 131, 271, 386 | | Version comparison scripts | `workflow/scripts/{pure_eb,cosebis,cl}_version_comparison.py` | | Covariance params | `workflow/rules/covariance.smk` line 4 (`get_cat_params`) | diff --git a/papers/bmodes/scripts/run_cov_sweep.sh b/papers/bmodes/scripts/run_cov_sweep.sh deleted file mode 100755 index f86f20dc..00000000 --- a/papers/bmodes/scripts/run_cov_sweep.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# CosmoCov integration-grid covariance sweep over non-fiducial versions (Design B). -# -# Loops the non-fiducial version list (resolved via sweep_versions.py) and runs -# the same run_cosmocov_chain.sh chain the fiducial cov_integration_g recipe -# calls, once per version, on the 1000-bin integration grid (Gaussian-only, -# masked, blind A). Each version's processed matrix lands in a per-version subdir -# named canonically so cosebis_version_comparison (--cov-dir) and the pure E/B -# sweep can reconstruct it: -# -# /covariance__A_g_minsep=0.5_maxsep=300.0_nbins=1000_masked/ -# covariance__A_g_minsep=0.5_maxsep=300.0_nbins=1000_masked_processed.txt -# -# run_cosmocov_chain.sh writes covariance_processed.txt into --out; this driver -# renames it to the {base}_processed.txt the plotter/pure_eb sweep expect. -# -# Mask per version: v1.4.8 uses the star-halo footprint power spectrum, every -# other version uses the standard footprint (mirrors covariance.smk -# get_mask_cls_path). Covariance is recomputed per variant — it is NOT -# correction-invariant (masked v1.4.8 differs; uncorrected σ_e differs slightly). -# -# Usage: -# run_cov_sweep.sh --config --cat-config \ -# --planck18-json /planck18.json \ -# --mask-base --out \ -# [--blind A] [--versions "v1 v2 ..."] -set -euo pipefail - -. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" - -CONFIG=""; CATCONFIG=""; PLANCK18=""; MASKBASE=""; OUT=""; BLIND="A"; VERSIONS="" -MINSEP=0.5; MAXSEP=300.0; NBINS=1000 -while [ $# -gt 0 ]; do - case "$1" in - --config) CONFIG="$2"; shift 2;; - --cat-config) CATCONFIG="$2"; shift 2;; - --planck18-json) PLANCK18="$2"; shift 2;; - --mask-base) MASKBASE="$2"; shift 2;; - --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; - --versions) VERSIONS="$2"; shift 2;; - *) echo "unknown arg: $1" >&2; exit 2;; - esac -done - -mkdir -p "$OUT" - -VERSIONS=$(sweep_versions "$CONFIG") - -for ver in $VERSIONS; do - base="covariance_${ver}_${BLIND}_g_minsep=${MINSEP}_maxsep=${MAXSEP}_nbins=${NBINS}_masked" - # Version dir key: strip SP_, _leak_corr, _ecutNN (mirrors get_mask_cls_path). - vdir=$(echo "$ver" | sed -e 's/_leak_corr//' -e 's/^SP_//' -e 's/_ecut[0-9]*//') - if [ "$vdir" = "v1.4.8" ]; then - mask="$MASKBASE/mask_cls_footprint_starhalo_nside_4096_norm.txt" - else - mask="$MASKBASE/mask_cls_footprint_nside_4096_norm.txt" - fi - echo "[cov_sweep] $ver (mask: $(basename "$mask"))" - bash "$WSCRIPTS/run_cosmocov_chain.sh" \ - --version "$ver" --blind "$BLIND" \ - --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" --gaussian g \ - --planck18-json "$PLANCK18" --cat-config "$CATCONFIG" \ - --mask-cls "$mask" --out "$OUT/$base" - mv "$OUT/$base/covariance_processed.txt" "$OUT/$base/${base}_processed.txt" - echo "[cov_sweep] $ver -> $OUT/$base/${base}_processed.txt" -done -echo "[cov_sweep] done -> $OUT" diff --git a/papers/bmodes/scripts/update_survey_stats.py b/papers/bmodes/scripts/update_survey_stats.py index 6713b3c0..a2dc386c 100644 --- a/papers/bmodes/scripts/update_survey_stats.py +++ b/papers/bmodes/scripts/update_survey_stats.py @@ -5,7 +5,7 @@ memory usage. Usage: - python workflow/scripts/update_survey_stats.py \ + python papers/bmodes/scripts/update_survey_stats.py \ --mask-standard --mask-starhalo """ diff --git a/papers/cosmo_val/Snakefile b/papers/cosmo_val/Snakefile index 716a3ac6..67bef0a7 100644 --- a/papers/cosmo_val/Snakefile +++ b/papers/cosmo_val/Snakefile @@ -7,7 +7,7 @@ # isolation or as the whole suite via the default `cosmo_val_all` target. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" +configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/pyproject.toml b/pyproject.toml index 5c45cfbf..4ad6fbf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,8 +159,8 @@ glass = [ # Snakemake workflow and cross-validation runners are available. workflow = [ "snakemake", - # run_2pcf_highres.py drives the MPI convergence run; the container ships - # OpenMPI under /opt/ompi so mpi4py builds against it. + # Optional MPI runners (the container ships OpenMPI at /opt/ompi, so mpi4py + # builds against it). "mpi4py", # CosmoSIS drives the cosmo_inference sampling step. Sdist-only, so this is a # source build (~2 min) against the base image's gfortran/GSL/cfitsio/OpenMPI — diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 861c3b05..275afc3c 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,55 +24,6 @@ rule xi: "../scripts/run_2pcf.py" -# PARKED: xi_highres (high-resolution xi for COSEBIS integration). Not runnable -# as written -- the shell invokes run_2pcf_highres.py bare, but the script has -# required --cat-config and --out arguments. Revive it with those supplied. -# -# The MPI reasoning below is hard-won and must survive the revival: -# -# Exception to the profile-driven container model: this is multi-node MPI, one -# `apptainer exec` per rank. Snakemake's own container wrapping puts the -# *whole* shell command -- `mpiexec` included -- inside a single container -# instance, so only rank 0's node would run inside it; the other ranks, -# spawned by SLURM/PMI on their own nodes, would land bare on the host. -# `container: None` plus an explicit `mpiexec -n N apptainer exec ...` -# per-rank is therefore required. -# Snakemake's slurm-jobstep plugin deliberately does NOT prepend `srun` to a -# job carrying an `mpi` resource, which is what lets the rule's own launcher -# run on the host, outside the container. -# Because this rule builds its own apptainer call, reaching the source-cache -# copy of the script relies on our `--bind /home` rather than on Snakemake's -# automatic mount -- and on a concrete image file, since `apptainer exec` -# takes no `docker://` URI. Take that path from `resolve_image()[0]` rather -# than naming a second image path that can drift. -# -# rule xi_highres: -# container: None -# params: -# image=resolve_image()[0], -# input: -# script=workflow.source_path("../scripts/run_2pcf_highres.py"), -# output: -# txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), -# xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), -# xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), -# resources: -# tasks=30, -# cpus_per_task=12, -# nodes=6, -# mem_mb_per_cpu=2000, -# runtime=2880, -# slurm_extra="'--exclude=n17,n09,n36 --partition=pscomp'", -# mpi="/softs/openmpi/5.0.5-slurm-CentOS8/bin/mpiexec", -# shell: -# "{resources.mpi} -n {resources.tasks} " -# "apptainer exec " -# "--bind /home,/n09data,/n17data,/n23data1,/softs " -# "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " -# "{params.image} " -# "python {input.script} --cat-config <...> --out <...>" - - rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py deleted file mode 100644 index eb31d2c7..00000000 --- a/workflow/scripts/run_2pcf_highres.py +++ /dev/null @@ -1,394 +0,0 @@ -#!/usr/bin/env python -""" -High-resolution ξ± measurement for COSEBIS integration. - -Computes TreeCorr GGCorrelation with fine angular binning (10,000+ bins) -required for accurate COSEBIS mode integration. Uses MPI for patch-pair -distribution across nodes when available; falls back to multi-threaded -single-process otherwise. - -Reference: Asgari et al. 2017 — minimum 10,000 bins for E_7 at 0.5% accuracy. - -Usage: - # MPI (via Slurm submission script): - mpiexec --map-by ppr:1:node python run_2pcf_highres.py \ - --cat-config /path/to/cosmo_val/cat_config.yaml --out - - # Single-process fallback: - python run_2pcf_highres.py \ - --cat-config /path/to/cosmo_val/cat_config.yaml --out -""" - -import argparse -import os -import time - -import numpy as np -import treecorr -from astropy.io import fits - -try: - # In-container path: full sp_validation stack available. - from sp_validation.cosmo_val import CosmologyValidation - - _HAVE_COSMO_VAL = True -except ImportError: - # Bare-host path (host OpenMPI + host python for the 10k-bin MPI run): the - # full sp_validation stack (cs_util.plots -> healpy/healsparse) is not - # installed. This measurement only needs the shear catalog path + column - # names, which are a pure cat_config.yaml lookup — resolve them standalone. - CosmologyValidation = None - _HAVE_COSMO_VAL = False - -# --------------------------------------------------------------------------- -# MPI setup (graceful fallback) -# --------------------------------------------------------------------------- -try: - from mpi4py import MPI - - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - size = comm.Get_size() - USE_MPI = size > 1 -except ImportError: - comm = None - rank = 0 - size = 1 - USE_MPI = False - -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- -# Shear response (R=1 for all SP catalogs) -R = 1.0 - -# Detect threads from Slurm or fall back to OS count -NUM_THREADS = int(os.environ.get("SLURM_CPUS_PER_TASK", os.cpu_count() or 24)) - -# The catalog path, ellipticity/weight columns, TreeCorr grid, patch count and -# output directory are resolved from the CLI in main() (defaults reproduce the -# historical hardcoded values for a no-arg run). They are declared here as -# module globals so the rank-aware helpers below resolve them at call time; the -# catalog path + columns come from cat_config + version exactly as run_2pcf.py -# resolves them (via CosmologyValidation). -CAT_PATH = None -VERSION = None -E1_COL = None -E2_COL = None -W_COL = None -TMIN = None # arcmin -TMAX = None # arcmin -NBINS = None -NPATCH = None -OUTPUT_DIR = None -PATCH_FILE = None - - -def parse_args(argv=None): - """CLI mirroring run_xi_sweep's signature; defaults reproduce prior behavior.""" - ap = argparse.ArgumentParser( - description="High-resolution TreeCorr ξ± measurement for COSEBIS integration." - ) - ap.add_argument( - "--config", - default=None, - help="Path to bmodes config.yaml (accepted for signature parity with " - "run_xi_sweep; not read by this measurement).", - ) - ap.add_argument( - "--cat-config", required=True, help="Absolute path to cat_config.yaml" - ) - ap.add_argument( - "--version", - default="SP_v1.4.6.3_leak_corr", - help="Catalog version key in cat_config", - ) - ap.add_argument("--nbins", type=int, default=10000, help="Number of log bins") - ap.add_argument("--npatch", type=int, default=50, help="TreeCorr patch count") - ap.add_argument( - "--min-sep", type=float, default=0.5, help="Min separation [arcmin]" - ) - ap.add_argument( - "--max-sep", type=float, default=300.0, help="Max separation [arcmin]" - ) - ap.add_argument("--out", required=True, help="Output directory (lc {output})") - return ap.parse_args(argv) - - -def log(msg): - """Print with timestamp on rank 0 only.""" - if rank == 0: - print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) - - -def load_catalog(): - """Load shear catalog and apply mean subtraction.""" - log(f"Loading catalog: {CAT_PATH}") - hdul = fits.open(CAT_PATH, memmap=True) - data = hdul[1].data - - ra = np.array(data["ra"], dtype=np.float64) - dec = np.array(data["dec"], dtype=np.float64) - e1 = np.array(data[E1_COL], dtype=np.float64) - e2 = np.array(data[E2_COL], dtype=np.float64) - w = np.array(data[W_COL], dtype=np.float64) - hdul.close() - - log(f" {len(ra):,} galaxies loaded") - - # Additive bias: c = _w (R=1 for SP catalogs) - c1 = np.average(e1 / R, weights=w) - c2 = np.average(e2 / R, weights=w) - log(f" Additive bias: c1={c1:.6e}, c2={c2:.6e}") - - # Calibrated shear: g = (e - c) / R - g1 = (e1 - c1) / R - g2 = (e2 - c2) / R - - return ra, dec, g1, g2, w - - -def _wait_for_file(path, timeout=300, interval=1.0): - """Block until `path` is visible to this node, defeating NFS dir caching. - - On a multi-node run the rank that wrote `path` sees it immediately, but - peer nodes can carry a stale negative directory-cache entry past an MPI - Barrier. Re-listing the parent directory forces an NFS attribute refresh; - poll that until the entry appears (or raise after `timeout` seconds). - """ - parent = os.path.dirname(path) or "." - name = os.path.basename(path) - waited = 0.0 - while waited < timeout: - try: - if name in os.listdir(parent): - return - except FileNotFoundError: - pass - time.sleep(interval) - waited += interval - raise TimeoutError(f"patch-center file not visible after {timeout}s: {path}") - - -def compute_patch_centers(ra, dec): - """Compute patch centers from subsampled catalog (rank 0 only).""" - if os.path.exists(PATCH_FILE): - log(f"Using existing patch centers: {PATCH_FILE}") - return - - if rank != 0: - return - - log(f"Computing patch centers (npatch={NPATCH}) from 1% subsample...") - rng = np.random.default_rng(42) - n_sub = max(len(ra) // 100, NPATCH * 100) - idx = rng.choice(len(ra), size=n_sub, replace=False) - - cat_sub = treecorr.Catalog( - ra=ra[idx], - dec=dec[idx], - ra_units="degrees", - dec_units="degrees", - npatch=NPATCH, - ) - cat_sub.write_patch_centers(PATCH_FILE) - log(f" Wrote patch centers to {PATCH_FILE}") - del cat_sub - - -def write_xi_fits(gg, prefix, xi_data): - """Write ξ+ or ξ- to FITS matching CosmologyValidation format.""" - out_path = os.path.join( - OUTPUT_DIR, - f"{prefix}_{VERSION}_minsep={TMIN}_maxsep={TMAX}_nbins={NBINS}_npatch=1.fits", - ) - n = len(xi_data) - cols = [ - fits.Column(name="BIN1", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="BIN2", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="ANGBIN", format="K", array=np.arange(1, n + 1)), - fits.Column(name="VALUE", format="D", array=xi_data), - fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr), - ] - ext_name = "XI_PLUS" if "plus" in prefix else "XI_MINUS" - hdu = fits.BinTableHDU.from_columns(cols, name=ext_name) - for key, val in { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = val - hdu.writeto(out_path, overwrite=True) - log(f" Wrote {out_path}") - - -def resolve_shear_config(cat_config_path, version): - """Standalone shear-config resolver (bare-host fallback for CosmologyValidation). - - Reproduces exactly the ``cc[version]["shear"]`` fields this measurement reads - (path, e1_col, e2_col, w_col), replicating CosmologyValidation's two - transforms: (1) subdir-relative path resolution, and (2) the ``_leak_corr`` - virtual version — deep-copy the base version and swap - e1_col/e2_col -> e1_col_corrected/e2_col_corrected. See - sp_validation/cosmo_val/core.py. - """ - import copy - - import yaml - - with open(cat_config_path) as fh: - cc = yaml.load(fh, Loader=yaml.FullLoader) - - def resolve_paths(ver): - subdir = os.fspath(cc[ver]["subdir"]) - for section in cc[ver].values(): - if isinstance(section, dict) and "path" in section: - p = section["path"] - if not os.path.isabs(p): - section["path"] = os.path.join(subdir, p) - - leak_suffix = "_leak_corr" - if version in cc: - resolve_paths(version) - elif version.endswith(leak_suffix): - base = version[: -len(leak_suffix)] - if base not in cc: - raise ValueError(f"Base version '{base}' not in cat_config for '{version}'") - resolve_paths(base) - base_shear = cc[base]["shear"] - if "e1_col_corrected" not in base_shear or "e2_col_corrected" not in base_shear: - raise ValueError( - f"{base} lacks e1_col_corrected/e2_col_corrected; cannot form {version}" - ) - cc[version] = copy.deepcopy(cc[base]) - cc[version]["shear"]["e1_col"] = base_shear["e1_col_corrected"] - cc[version]["shear"]["e2_col"] = base_shear["e2_col_corrected"] - resolve_paths(version) - else: - raise ValueError(f"Version '{version}' not found in cat_config") - - return cc[version]["shear"] - - -def main(): - global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL - global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE - - args = parse_args() - VERSION = args.version - NBINS = args.nbins - NPATCH = args.npatch - TMIN = args.min_sep - TMAX = args.max_sep - OUTPUT_DIR = args.out - - # Resolve catalog path + ellipticity/weight columns from cat_config + version - # exactly as run_2pcf.py does (applies the _leak_corr column swap and the - # subdir path resolution). In-container this uses CosmologyValidation; bare- - # host (10k-bin MPI run) it uses the standalone cat_config resolver, which is - # byte-identical for the shear-config fields this measurement reads. - if _HAVE_COSMO_VAL: - cv = CosmologyValidation( - versions=[VERSION], catalog_config=args.cat_config, output_dir=OUTPUT_DIR - ) - shear_cfg = cv.cc[VERSION]["shear"] - else: - shear_cfg = resolve_shear_config(args.cat_config, VERSION) - CAT_PATH = shear_cfg["path"] - E1_COL = shear_cfg["e1_col"] - E2_COL = shear_cfg["e2_col"] - W_COL = shear_cfg["w_col"] - - PATCH_FILE = os.path.join( - OUTPUT_DIR, - f"patch_centers_{VERSION}_{NPATCH}_{TMIN}_{TMAX}.dat", - ) - - t0 = time.time() - - log("=" * 60) - log("High-resolution ξ± measurement") - log(f" MPI: {'yes' if USE_MPI else 'no'} (ranks={size})") - log(f" Config: {NBINS:,} bins, [{TMIN}, {TMAX}] arcmin") - log(f" Patches: {NPATCH}, Threads/rank: {NUM_THREADS}") - log(f" Version: {VERSION}") - log("=" * 60) - - # All ranks load catalog (needed for TreeCorr patch assignment) - ra, dec, g1, g2, w = load_catalog() - - # Compute patch centers (rank 0 only; others wait) - compute_patch_centers(ra, dec) - if USE_MPI: - comm.Barrier() - # Cross-node visibility: rank 0 wrote PATCH_FILE on its node, but on a - # multi-node allocation the other ranks' nodes may not see it yet (NFS - # close-to-open + negative-dir caching persists past the Barrier). Poll - # with a forced directory refresh until it appears before reading it. - _wait_for_file(PATCH_FILE) - - # Create TreeCorr catalog with patch centers - log("Creating TreeCorr catalog with patches...") - cat = treecorr.Catalog( - ra=ra, - dec=dec, - g1=g1, - g2=g2, - w=w, - ra_units="degrees", - dec_units="degrees", - patch_centers=PATCH_FILE, - ) - cat.load() - cat.get_patches() - log(f" Catalog ready ({cat.nobj:,} objects, {cat.npatch} patches)") - - # Free raw arrays (TreeCorr holds its own copy) - del ra, dec, g1, g2, w - - # Compute GG correlation - log("Computing GGCorrelation...") - gg = treecorr.GGCorrelation( - min_sep=TMIN, - max_sep=TMAX, - nbins=NBINS, - sep_units="arcminutes", - verbose=2, - ) - - process_kwargs = {"num_threads": NUM_THREADS} - if USE_MPI: - process_kwargs["comm"] = comm - - gg.process(cat, **process_kwargs) - log(f" Correlation complete ({time.time() - t0:.0f}s elapsed)") - - # Write output (rank 0 only) - if rank == 0: - out_txt = os.path.join( - OUTPUT_DIR, - f"{VERSION}_xi_minsep={TMIN}_maxsep={TMAX}_nbins={NBINS}_npatch=1.txt", - ) - # Write only the main per-bin correlation. The convergence consumer - # (cosebis_binning_comparison.py) reads just the per-bin columns - # (np.loadtxt max_rows=nbins) and the 1000-bin covariance — the 10k - # jackknife cov is used nowhere. write_patch_results/write_cov=True - # serialised a 20000x20000 cov + 180 patch blocks (~10 GB) that nothing - # reads and also cost the estimate_cov compute; drop both. The patches - # still parallelise gg.process; gg.xip/gg.xim (values, FITS) are - # unaffected. - gg.write(out_txt, write_patch_results=False, write_cov=False) - log(f" Wrote {out_txt}") - - write_xi_fits(gg, "xi_plus", gg.xip) - write_xi_fits(gg, "xi_minus", gg.xim) - - elapsed = time.time() - t0 - log(f"Done! Total time: {elapsed / 3600:.1f}h ({elapsed:.0f}s)") - - -if __name__ == "__main__": - main() From 0f2f2c94e6f83a7dc43f08c8c9c19c574bd153ed Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 15:15:46 +0200 Subject: [PATCH 26/83] Work b_modes from plain arrays instead of TreeCorr correlations calculate_pure_eb_correlation returns a self-describing results dict (theta, bin edges, xi+/- on both grids, n_eff) in place of the gg/gg_int objects, so calculate_eb_statistics, the plot helpers and save_pure_eb_results work from values alone. New seams: bins_from_edges, log_bin_edges, pure_eb_from_xi, pure_eb_covariance_mc and cosebis_scan_from_xi. The pure-E/B .npz stores n_eff (the realisation count behind the covariance) instead of npatch. Callers in cosmo_val, the bmodes calculate_pure_eb_ptes script, its claims rule and PTE sweep driver are adapted. No numbers change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR --- papers/bmodes/rules/claims.smk | 14 +- .../bmodes/scripts/calculate_pure_eb_ptes.py | 35 +- .../bmodes/scripts/run_pure_eb_ptes_sweep.sh | 26 +- src/sp_validation/b_modes.py | 455 +++++++++++------- src/sp_validation/cosmo_val/core.py | 6 +- src/sp_validation/cosmo_val/cosebis.py | 2 +- src/sp_validation/cosmo_val/pure_eb.py | 24 +- src/sp_validation/tests/test_b_modes.py | 161 ++++++- src/sp_validation/tests/test_cosmo_val.py | 2 +- 9 files changed, 467 insertions(+), 258 deletions(-) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index 43962627..3846b9bc 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -355,25 +355,21 @@ rule pure_eb_covariance: rule calculate_pure_eb_ptes: - """Calculate PTE matrices for Pure E/B mode scale cut robustness. + """PTE matrices for pure E/B-mode scale-cut robustness. - Per-blind: Uses blind-specific integration covariance for PTE calculation. - The pure_eb_data vectors are identical across blinds; only covariance differs. - - In practice, BB covariance is blind-independent (validated by - bb_covariance_blind_independence), so downstream consumers (config_space_pte_matrices) - only request blind A. The per-blind wildcard is retained for the blind independence test. + Nothing here varies with the blind: the data vectors come from the blind-A + gather and the PTEs are Hartlap-debiased by the MC draw count, not by a + per-blind covariance. The wildcard survives as the filename slot the + consumer (config_space_pte_matrices) reads, and only blind A is ever built. """ input: pure_eb_data="results/paper_plots/intermediate/{version}_A_pure_eb_semianalytic.npz", - cov_integration=lambda w: _cov_integration_path(w.version, w.blind), output: "results/paper_plots/intermediate/{version}_{blind}_pure_eb_ptes.npz", wildcard_constraints: blind=r"[ABC]", params: version="{version}", - npatch=FIDUCIAL["npatch"], n_samples=config["covariance"]["n_samples"], resources: mem_mb=16000, diff --git a/papers/bmodes/scripts/calculate_pure_eb_ptes.py b/papers/bmodes/scripts/calculate_pure_eb_ptes.py index f8bc709d..3a623669 100644 --- a/papers/bmodes/scripts/calculate_pure_eb_ptes.py +++ b/papers/bmodes/scripts/calculate_pure_eb_ptes.py @@ -4,14 +4,13 @@ pure-E/B ``semianalytic.npz`` (data vectors + MC covariance), evaluates the ξ_+^B / ξ_-^B / joint ξ_tot^B χ² PTE matrices over the scale-cut grid via ``sp_validation.b_modes.calculate_eb_statistics`` (Hartlap-corrected inverse -MC covariance), and writes the PTE matrices to +MC covariance, debiased by the draw count), and writes the PTE matrices to ``{out}/{version}_{blind}_pure_eb_ptes.npz``. python calculate_pure_eb_ptes.py \ --version SP_v1.4.6.3_leak_corr --blind A \ --pure-eb-data <..._pure_eb_semianalytic.npz> \ - --cov-integration \ - --npatch 1 --n-samples 2000 --out + --n-samples 2000 --out """ import argparse @@ -22,31 +21,21 @@ from sp_validation.b_modes import calculate_eb_statistics -class FakeGG: - """Minimal GGCorrelation-like object for calculate_eb_statistics.""" - - def __init__(self, nbins, npatch): - self.nbins = nbins - self.npatch1 = npatch - self.npatch2 = npatch - - def calculate_ptes( version, blind, pure_eb_data, - cov_integration, - npatch, n_samples, output_dir, ): dataset = np.load(pure_eb_data) theta = dataset["theta"] - nbins = len(theta) results = { - "gg": FakeGG(nbins, int(npatch)), + "theta": theta, + # The MC draws are the realisations behind this covariance. + "n_eff": int(n_samples), "xip_E": dataset["xip_E"], "xim_E": dataset["xim_E"], "xip_B": dataset["xip_B"], @@ -57,11 +46,7 @@ def calculate_ptes( } print(f"Calculating PTE matrices for {version}...") - results = calculate_eb_statistics( - results, - cov_path_int=cov_integration, - n_samples=int(n_samples), - ) + results = calculate_eb_statistics(results) pte_matrices = results["pte_matrices"] output_data = { @@ -83,12 +68,6 @@ def _from_cli(argv=None): ap.add_argument("--version", required=True) ap.add_argument("--blind", default="A") ap.add_argument("--pure-eb-data", required=True, help="Gathered semianalytic .npz") - ap.add_argument( - "--cov-integration", - default=None, - help="Integration-grid covariance _processed.txt (optional)", - ) - ap.add_argument("--npatch", type=int, default=1) ap.add_argument("--n-samples", type=int, default=2000) ap.add_argument("--out", required=True, help="Output directory (lc {output})") a = ap.parse_args(argv) @@ -96,8 +75,6 @@ def _from_cli(argv=None): version=a.version, blind=a.blind, pure_eb_data=a.pure_eb_data, - cov_integration=a.cov_integration, - npatch=a.npatch, n_samples=a.n_samples, output_dir=a.out, ) diff --git a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh index 0ea3a862..0e7ecdf5 100644 --- a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh @@ -4,30 +4,27 @@ # Loops the non-fiducial version list (resolved via sweep_versions.py) and runs # calculate_pure_eb_ptes.py once per version — the same χ² PTE-matrix compute the # fiducial pure_eb_pte_per_cut recipe calls. Each version's gathered semi-analytic -# NPZ is read from the pure_eb_sweep output dir and its Gaussian integration -# covariance from the cov_sweep output dir (both by absolute path — lc does not -# wire cross-output deps, so the driver reads upstream sweeps directly; run -# pure_eb_sweep + cov_sweep before this). Each version emits the canonical -# ``{ver}_{blind}_pure_eb_ptes.npz`` — the exact name config_space_pte_matrices.py -# reconstructs from --pte-intermediate-dir — straight into --out. Serial over -# versions; each version is fast (206-pair grid, ~seconds). +# NPZ (data vectors + MC covariance) is read from the pure_eb_sweep output dir by +# absolute path — lc does not wire cross-output deps, so the driver reads the +# upstream sweep directly; run pure_eb_sweep before this. Each version emits the +# canonical ``{ver}_{blind}_pure_eb_ptes.npz`` — the exact name +# config_space_pte_matrices.py reconstructs from --pte-intermediate-dir — straight +# into --out. Serial over versions; each version is fast (206-pair grid, ~seconds). # # Usage: # run_pure_eb_ptes_sweep.sh --config --cat-config \ # --pure-eb-sweep-dir \ -# --cov-sweep-dir \ # --out [--blind A] [--versions "v1 v2 ..."] set -euo pipefail . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" +CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" while [ $# -gt 0 ]; do case "$1" in --config) CONFIG="$2"; shift 2;; --cat-config) CATCONFIG="$2"; shift 2;; --pure-eb-sweep-dir) PUREEBSWEEP="$2"; shift 2;; - --cov-sweep-dir) COVSWEEP="$2"; shift 2;; --out) OUT="$2"; shift 2;; --blind) BLIND="$2"; shift 2;; --versions) VERSIONS="$2"; shift 2;; @@ -41,16 +38,11 @@ VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do pureeb="$PUREEBSWEEP/${ver}_${BLIND}_pure_eb_semianalytic.npz" - covbase="covariance_${ver}_${BLIND}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" - covint="$COVSWEEP/$covbase/${covbase}_processed.txt" - for f in "$pureeb" "$covint"; do - [ -f "$f" ] || { echo "MISSING upstream input for $ver: $f" >&2; exit 1; } - done + [ -f "$pureeb" ] || { echo "MISSING upstream input for $ver: $pureeb" >&2; exit 1; } echo "[pure_eb_ptes_sweep] $ver" spv_python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ --version "$ver" --blind "$BLIND" \ - --pure-eb-data "$pureeb" --cov-integration "$covint" \ - --npatch 1 --n-samples 2000 --out "$OUT" + --pure-eb-data "$pureeb" --n-samples 2000 --out "$OUT" echo "[pure_eb_ptes_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_ptes.npz" done echo "[pure_eb_ptes_sweep] done -> $OUT" diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 13f4c2e9..c4b9a2b7 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -74,23 +74,34 @@ def scale_cut_to_bins(gg, min_scale=None, max_scale=None): stop_bin : int Last included bin index + 1 (for slicing notation) """ - nbins = len(gg.meanr) + return bins_from_edges(gg.left_edges, gg.right_edges, min_scale, max_scale) - if min_scale is not None: - # Conservative: exclude bins whose left edge is below min_scale - start_bin = np.searchsorted(gg.left_edges, min_scale, side="left") - else: - start_bin = 0 - - if max_scale is not None: - # Conservative: exclude bins whose right edge is above max_scale - stop_bin = np.searchsorted(gg.right_edges, max_scale, side="right") - else: - stop_bin = nbins +def bins_from_edges(left_edges, right_edges, min_scale=None, max_scale=None): + """:func:`scale_cut_to_bins` on bare bin edges.""" + start_bin = ( + np.searchsorted(left_edges, min_scale, side="left") + if min_scale is not None + else 0 + ) + stop_bin = ( + np.searchsorted(right_edges, max_scale, side="right") + if max_scale is not None + else len(right_edges) + ) return start_bin, stop_bin +def log_bin_edges(min_sep, max_sep, nbins): + """TreeCorr ``Log`` bin edges — the grid a binning defines. + + SACC ξ± parts store bin centres, not edges, so a consumer working from a + part reconstructs the edges from the binning it was measured on. + """ + edges = np.geomspace(float(min_sep), float(max_sep), int(nbins) + 1) + return edges[:-1], edges[1:] + + def correlation_from_covariance(covariance): """ Convert covariance matrix to correlation matrix. @@ -164,75 +175,40 @@ def pure_EB(corrs): parallel=True, ) - # Initialize results dictionary with basic E/B mode data - results = {"gg": gg, "gg_int": gg_int} + # The results dict is self-describing: the grids it was measured on travel + # with the modes, so every consumer downstream works from values alone. + results = { + "theta": gg.meanr, + "left_edges": gg.left_edges, + "right_edges": gg.right_edges, + "xip": gg.xip, + "xim": gg.xim, + "var_xip": gg.varxip, + "var_xim": gg.varxim, + "theta_int": gg_int.meanr, + "xip_int": gg_int.xip, + "xim_int": gg_int.xim, + "n_eff": n_samples if cov_path_int is not None else gg.npatch1, + } results.update(dict(zip(_EB_KEYS, pure_EB([gg, gg_int])))) if cov_path_int is not None: - # Use semi-analytical covariance propagation - print("Computing semi-analytical covariance for pure E/B modes") - if z_dist is None: - raise ValueError("z_dist must be provided for semi-analytical covariance") - if cosmo_cov is None: + if z_dist is None or cosmo_cov is None: raise ValueError( - "cosmo_cov must be provided for semi-analytical covariance" - ) - - # Load covariance matrix - cov_int = np.loadtxt(cov_path_int) - - # Set up integration binning and pre-compute binning matrix - nbins_int, theta_int = len(gg_int.meanr), gg_int.meanr - reporting_bin_edges = np.concatenate([gg.left_edges, [gg.right_edges[-1]]]) - bin_indices = np.digitize(theta_int, reporting_bin_edges) - 1 - - valid_mask = (bin_indices >= 0) & (bin_indices < len(gg.meanr)) - row_indices, col_indices = (bin_indices[valid_mask], np.where(valid_mask)[0]) - - binning_matrix = sparse.csr_matrix( - (np.ones(len(row_indices)), (row_indices, col_indices)), - shape=(len(gg.meanr), nbins_int), - ) - row_sums = np.array(binning_matrix.sum(axis=1)).flatten() - binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix - - # Generate theoretical xi+/xi- predictions and sample - mean_int = np.concatenate( - get_theo_xi( - theta=theta_int, - z=z_dist[:, 0], - nz=z_dist[:, 1], - backend="ccl", - cosmo=cosmo_cov, + "semi-analytical covariance needs both z_dist and cosmo_cov" ) + cov, eb_samples = pure_eb_covariance_mc( + theta=gg.meanr, + left_edges=gg.left_edges, + right_edges=gg.right_edges, + theta_int=gg_int.meanr, + cov_int=np.loadtxt(cov_path_int), + z=z_dist[:, 0], + nz=z_dist[:, 1], + cosmo=cosmo_cov, + n_samples=n_samples, ) - - samples_int = np.random.multivariate_normal(mean_int, cov_int, size=n_samples) - samples_int_xip = samples_int[:, :nbins_int] - samples_int_xim = samples_int[:, nbins_int:] - samples_rep_xip = (binning_matrix @ samples_int_xip.T).T - samples_rep_xim = (binning_matrix @ samples_int_xim.T).T - - transformed_samples = [ - np.concatenate( - get_pure_EB_modes( - theta=gg.meanr, - theta_int=gg_int.meanr, - xip=samples_rep_xip[i], - xim=samples_rep_xim[i], - xip_int=samples_int_xip[i], - xim_int=samples_int_xim[i], - tmin=min_sep, - tmax=max_sep, - parallel=True, - ) - ) - for i in tqdm.tqdm(range(n_samples), desc="MC samples") - ] - - # Store semi-analytical covariance results - eb_samples = np.array(transformed_samples) - results.update({"cov": np.cov(eb_samples.T), "eb_samples": eb_samples}) + results.update({"cov": cov, "eb_samples": eb_samples}) else: # Use existing treecorr covariance estimation results["cov"] = treecorr.estimate_multi_cov( @@ -255,6 +231,112 @@ def pure_EB(corrs): return results +def pure_eb_from_xi( + theta_report, xip_report, xim_report, theta_int, xip_int, xim_int, tmin, tmax +): + """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. + + The values-only seam of :func:`calculate_pure_eb_correlation`, for callers + holding ξ± arrays rather than TreeCorr correlations. + + ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* + (``gg.left_edges[0]`` / ``gg.right_edges[-1]``). The reporting grid must be a + strict sub-range of the integration grid: a reporting point on the + integration boundary has no interior support and comes back NaN. + + Returns + ------- + dict + Keyed by ``_EB_KEYS`` (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb). + """ + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes + + modes = get_pure_EB_modes( + theta=np.asarray(theta_report), + xip=np.asarray(xip_report), + xim=np.asarray(xim_report), + theta_int=np.asarray(theta_int), + xip_int=np.asarray(xip_int), + xim_int=np.asarray(xim_int), + tmin=tmin, + tmax=tmax, + parallel=True, + ) + return dict(zip(_EB_KEYS, (np.asarray(m) for m in modes))) + + +def pure_eb_covariance_mc( + *, + theta, + left_edges, + right_edges, + theta_int, + cov_int, + z, + nz, + cosmo, + n_samples=1000, +): + """Pure-E/B covariance by Monte Carlo through the same kernel as the modes. + + ξ± draws come from ``cov_int``, a ξ± covariance on the integration grid, + around the theory mean for ``(z, nz)`` under ``cosmo``; each draw is binned + down to the reporting grid and pushed through ``get_pure_EB_modes``. The + covariance of the transformed draws is the result, so it depends on the + covariance model and the grids, never on the measured data vector. + + Returns ``(cov, eb_samples)`` — the covariance in ``_EB_KEYS`` order and + the draws behind it. + """ + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes + + theta, theta_int = np.asarray(theta), np.asarray(theta_int) + nbins_int = len(theta_int) + + # Each reporting bin averages the integration bins that fall inside it. + reporting_bin_edges = np.concatenate([left_edges, [right_edges[-1]]]) + bin_indices = np.digitize(theta_int, reporting_bin_edges) - 1 + valid_mask = (bin_indices >= 0) & (bin_indices < len(theta)) + row_indices, col_indices = (bin_indices[valid_mask], np.where(valid_mask)[0]) + binning_matrix = sparse.csr_matrix( + (np.ones(len(row_indices)), (row_indices, col_indices)), + shape=(len(theta), nbins_int), + ) + row_sums = np.array(binning_matrix.sum(axis=1)).flatten() + binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix + + mean_int = np.concatenate( + get_theo_xi(theta=theta_int, z=z, nz=nz, backend="ccl", cosmo=cosmo) + ) + samples_int = np.random.multivariate_normal(mean_int, cov_int, size=n_samples) + samples_int_xip, samples_int_xim = ( + samples_int[:, :nbins_int], + samples_int[:, nbins_int:], + ) + samples_rep_xip = (binning_matrix @ samples_int_xip.T).T + samples_rep_xim = (binning_matrix @ samples_int_xim.T).T + + eb_samples = np.array( + [ + np.concatenate( + get_pure_EB_modes( + theta=theta, + theta_int=theta_int, + xip=samples_rep_xip[i], + xim=samples_rep_xim[i], + xip_int=samples_int_xip[i], + xim_int=samples_int_xim[i], + tmin=left_edges[0], + tmax=right_edges[-1], + parallel=True, + ) + ) + for i in tqdm.tqdm(range(n_samples), desc="MC samples") + ] + ) + return np.cov(eb_samples.T), eb_samples + + def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. @@ -279,23 +361,52 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): Each results dictionary contains 'En', 'Bn', 'cov', 'chi2_E', 'chi2_B', 'pte_B', 'scale_cut', and 'mask' entries. """ - from cosmo_numba.B_modes.cosebis import COSEBIS + cov_xipm = np.loadtxt(cov_path) if cov_path is not None else gg.cov + return cosebis_scan_from_xi( + gg.meanr, + gg.xip, + gg.xim, + cov_xipm, + gg.left_edges, + gg.right_edges, + nmodes=nmodes, + scale_cuts=scale_cuts, + # A theory covariance has no jackknife realisations to debias. + npatch=None if cov_path is not None else gg.npatch1, + ) - # Default to full range if no scale cuts provided - if scale_cuts is None: - scale_cuts = [(gg.left_edges[0], gg.right_edges[-1])] - # Pre-compute values that don't change across scale cuts - nbins = len(gg.meanr) +def cosebis_scan_from_xi( + theta, + xip, + xim, + cov_xipm, + left_edges, + right_edges, + *, + nmodes=10, + scale_cuts=None, + npatch=None, +): + """COSEBIs over a set of scale cuts, from ξ± arrays and their covariance. - # Load covariance matrix and calculate Hartlap factor once - if cov_path is not None: - print(f"Loading theoretical covariance from {cov_path}") - cov_xipm = np.loadtxt(cov_path) - hartlap_factor = 1 # Not defined for analytic covariances - else: - cov_xipm = gg.cov - hartlap_factor = (gg.npatch1 - 2 * nmodes - 2) / (gg.npatch1 - 1) + The values-and-covariance seam of :func:`calculate_cosebis`, for callers + holding a ξ± data vector rather than a TreeCorr ``GGCorrelation``. The + COSEBIs covariance is the ξ± covariance carried through the same linear + kernel as the modes, so no estimator re-run is involved. ``npatch`` is the + jackknife realisation count behind ``cov_xipm``, which sets the Hartlap + debiasing; leave it ``None`` for a theory covariance, which needs none. + + Returns the ``{scale_cut: result}`` mapping :func:`calculate_cosebis` returns. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + + theta, xip, xim = (np.asarray(a) for a in (theta, xip, xim)) + cov_xipm = np.asarray(cov_xipm) + if scale_cuts is None: + scale_cuts = [(left_edges[0], right_edges[-1])] + nbins = len(theta) + hartlap_factor = 1 if npatch is None else (npatch - 2 * nmodes - 2) / (npatch - 1) all_results = {} @@ -303,11 +414,12 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): for scale_cut in tqdm.tqdm(scale_cuts, desc="COSEBIs scale cuts"): min_theta, max_theta = scale_cut - # Apply scale cuts using scale_cut_to_bins for consistency - start_bin, stop_bin = scale_cut_to_bins(gg, min_theta, max_theta) + start_bin, stop_bin = bins_from_edges( + left_edges, right_edges, min_theta, max_theta + ) inds = np.arange(start_bin, stop_bin) - theta_cut, xip_cut, xim_cut = [arr[inds] for arr in [gg.meanr, gg.xip, gg.xim]] + theta_cut, xip_cut, xim_cut = [arr[inds] for arr in [theta, xip, xim]] # Calculate COSEBIs E/B modes using actual theta range (per Axel's recommendation) # Use precision=120 (vs default 80) to avoid sympy root convergence failures @@ -354,11 +466,7 @@ def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): return all_results -def calculate_eb_statistics( - results, - cov_path_int=None, - n_samples=1000, -): +def calculate_eb_statistics(results): """ Calculate E/B mode statistics using 2D PTE analysis for all scale cut combinations. @@ -369,23 +477,17 @@ def calculate_eb_statistics( Parameters ---------- results : dict - Dictionary containing pure E/B mode results from calculate_pure_eb_correlation - cov_path_int : str, optional - Path to integration covariance matrix for semi-analytical calculation - n_samples : int, optional - Number of Monte Carlo samples used for semi-analytical covariance - min_bins : int, optional - Minimum number of bins required for valid PTE calculation + Pure E/B results: the six mode arrays, the ``cov`` block, the reporting + ``theta``, and ``n_eff`` — the realisation count behind the covariance + (jackknife patches or MC draws), which sets the Hartlap debiasing Returns ------- dict Updated results dictionary with PTE matrices and statistics """ - gg = results["gg"] - nbins = gg.nbins - npatch = gg.npatch1 - n_eff = n_samples if cov_path_int is not None else npatch + nbins = len(results["theta"]) + n_eff = results["n_eff"] # Extract covariance blocks and standard deviations cov = results["cov"] @@ -449,16 +551,16 @@ def calculate_eb_statistics( return results -def plot_integration_vs_reporting(gg, gg_int, output_path, version): +def plot_integration_vs_reporting(results, output_path, version): """ Plot integration vs reporting scale comparison. Parameters ---------- - gg : treecorr.GGCorrelation - Reporting scale correlation function - gg_int : treecorr.GGCorrelation - Integration scale correlation function + results : dict + Pure E/B results carrying both grids (``theta``/``xip``/``xim`` and the + ``theta_int``/``xip_int``/``xim_int`` counterparts), plus the reporting + ``var_xip``/``var_xim`` the error bars use output_path : str Output file path for the plot version : str @@ -468,26 +570,24 @@ def plot_integration_vs_reporting(gg, gg_int, output_path, version): # Configure plot data for both xi+ and xi- in a consolidated loop plot_configs = [ - ("+", "xip", "varxip", r"$\theta \xi_+(\theta) \times 10^4$"), - ("-", "xim", "varxim", r"$\theta \xi_-(\theta) \times 10^4$"), + ("+", "xip", r"$\theta \xi_+(\theta) \times 10^4$"), + ("-", "xim", r"$\theta \xi_-(\theta) \times 10^4$"), ] data_configs = [ - (gg_int, "k.", 3, 0.3, "Integration"), - (gg, ".", 12, 1, "Reporting"), + ("_int", "k.", 3, 0.3, "Integration"), + ("", ".", 12, 1, "Reporting"), ] - for ax_idx, (xi_label, xi_attr, var_attr, ylabel) in enumerate(plot_configs): - for data, fmt, ms, alpha, label_type in data_configs: - xi_val = getattr(data, xi_attr) - yerr = ( - data.meanr * np.sqrt(getattr(data, var_attr)) / 1e-4 - if hasattr(data, var_attr) and label_type == "Reporting" - else None - ) + for ax_idx, (xi_label, xi_attr, ylabel) in enumerate(plot_configs): + for suffix, fmt, ms, alpha, label_type in data_configs: + theta = results[f"theta{suffix}"] + xi_val = results[f"{xi_attr}{suffix}"] + var = results.get(f"var_{xi_attr}{suffix}") + yerr = theta * np.sqrt(var) / 1e-4 if var is not None else None axs[ax_idx].errorbar( - data.meanr, - data.meanr * xi_val / 1e-4, + theta, + theta * xi_val / 1e-4, yerr=yerr, fmt=fmt, ms=ms, @@ -496,8 +596,8 @@ def plot_integration_vs_reporting(gg, gg_int, output_path, version): ls="" if label_type == "Reporting" else None, label=( rf"$\xi_{{{xi_label}}}$, {label_type}: " - rf"${data.min_sep} < \theta < {data.max_sep}$, " - rf"{data.nbins} bins" + rf"${theta[0]:.2g} < \theta < {theta[-1]:.4g}$, " + rf"{len(theta)} bins" ), ) axs[ax_idx].set( @@ -513,7 +613,7 @@ def plot_integration_vs_reporting(gg, gg_int, output_path, version): plt.savefig(output_path, dpi=300, bbox_inches="tight") -def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): +def _get_pte_from_scale_cut(pte_matrix, edges, scale_cut): """ Extract PTE value from matrix based on scale cut range using conservative logic. @@ -521,8 +621,8 @@ def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): ---------- pte_matrix : numpy.ndarray 2D PTE matrix - gg : treecorr.GGCorrelation - Correlation function object with bin edges + edges : tuple of numpy.ndarray + ``(left_edges, right_edges)`` of the grid the matrix is indexed on scale_cut : tuple (min_scale, max_scale) angular range for scale cut @@ -531,7 +631,8 @@ def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): float PTE value for the given scale cut, or full-range PTE if scale_cut is None """ - nbins = len(gg.meanr) + left_edges, right_edges = edges + nbins = len(left_edges) if scale_cut is None: # Return full-range PTE (first row, last column) @@ -539,8 +640,7 @@ def _get_pte_from_scale_cut(pte_matrix, gg, scale_cut): min_scale, max_scale = scale_cut - # Use conservative scale_cut_to_bins helper - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges(left_edges, right_edges, min_scale, max_scale) # Ensure valid range, otherwise fallback to full range if stop_bin <= start_bin or start_bin >= nbins or stop_bin <= 0: @@ -571,19 +671,20 @@ def plot_pure_eb_correlations( fiducial_xim_scale_cut : tuple, optional (min_scale, max_scale) for xi- fiducial analysis, shown as gray regions """ - gg = results["gg"] - nbins = gg.nbins + theta = results["theta"] + edges = (results["left_edges"], results["right_edges"]) + nbins = len(theta) cov = results["cov"] # Calculate combined PTE using off-diagonal covariance blocks # Get scale cuts for both xi+ and xi- if fiducial_xip_scale_cut is not None: - xip_start_bin, xip_stop_bin = scale_cut_to_bins(gg, *fiducial_xip_scale_cut) + xip_start_bin, xip_stop_bin = bins_from_edges(*edges, *fiducial_xip_scale_cut) else: xip_start_bin, xip_stop_bin = 0, nbins if fiducial_xim_scale_cut is not None: - xim_start_bin, xim_stop_bin = scale_cut_to_bins(gg, *fiducial_xim_scale_cut) + xim_start_bin, xim_stop_bin = bins_from_edges(*edges, *fiducial_xim_scale_cut) else: xim_start_bin, xim_stop_bin = 0, nbins @@ -618,7 +719,7 @@ def plot_pure_eb_correlations( if "eb_samples" in results: # Semi-analytical case n_eff = results["eb_samples"].shape[0] else: # Jackknife case - n_eff = gg.npatch1 + n_eff = results["n_eff"] hartlap_factor = (n_eff - nbins_eff - 2) / (n_eff - 1) chi2_combined = hartlap_factor * ( @@ -628,10 +729,10 @@ def plot_pure_eb_correlations( # Extract PTE values for fiducial scale cuts (or full range) xip_B_pte = _get_pte_from_scale_cut( - results["pte_matrices"]["xip_B"], gg, fiducial_xip_scale_cut + results["pte_matrices"]["xip_B"], edges, fiducial_xip_scale_cut ) xim_B_pte = _get_pte_from_scale_cut( - results["pte_matrices"]["xim_B"], gg, fiducial_xim_scale_cut + results["pte_matrices"]["xim_B"], edges, fiducial_xim_scale_cut ) fig, axs = plt.subplots(1, 2, figsize=(14, 6), sharex=True, sharey=True) @@ -643,14 +744,14 @@ def plot_pure_eb_correlations( ( "xip", "+", - "varxip", + "var_xip", r"$\xi_{+}=\xi_{+}^{E}+\xi_{+}^{B}+\xi_{+}^{\mathrm{amb}}$", xip_B_pte, ), ( "xim", "-", - "varxim", + "var_xim", r"$\xi_{-}=\xi_{-}^{E}-\xi_{-}^{B}+\xi_{-}^{\mathrm{amb}}$", xim_B_pte, ), @@ -660,11 +761,11 @@ def plot_pure_eb_correlations( plot_configs ): # Plot main correlation function - xi_val = getattr(gg, xi_type) + xi_val = results[xi_type] axs[ax_idx].errorbar( - gg.meanr, - gg.meanr * xi_val / scale_factor, - yerr=gg.meanr * np.sqrt(getattr(gg, var_attr)) / scale_factor, + theta, + theta * xi_val / scale_factor, + yerr=theta * np.sqrt(results[var_attr]) / scale_factor, fmt="k.", capsize=3, label=main_label, @@ -690,9 +791,9 @@ def plot_pure_eb_correlations( for key, color, alpha, label in plot_data: axs[ax_idx].errorbar( - gg.meanr, - gg.meanr * results[key] / scale_factor, - yerr=gg.meanr * results[f"std_{key}"] / scale_factor, + theta, + theta * results[key] / scale_factor, + yerr=theta * results[f"std_{key}"] / scale_factor, color=color, ls="", marker=".", @@ -722,12 +823,12 @@ def plot_pure_eb_correlations( xlim = original_xlims[ax_idx] # Use conservative scale_cut_to_bins helper for consistency - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges(*edges, min_scale, max_scale) # Show excluded regions based on bin edges used in PTE calculation # Lower exclusion: bins 0 to start_bin-1 are excluded if start_bin > 0: - lower_exclusion_edge = gg.right_edges[start_bin - 1] + lower_exclusion_edge = edges[1][start_bin - 1] axs[ax_idx].axvspan( xlim[0], lower_exclusion_edge, @@ -737,8 +838,8 @@ def plot_pure_eb_correlations( ) # Upper exclusion: bins stop_bin to end are excluded - if stop_bin < len(gg.left_edges): - upper_exclusion_edge = gg.left_edges[stop_bin] + if stop_bin < len(edges[0]): + upper_exclusion_edge = edges[0][stop_bin] axs[ax_idx].axvspan( upper_exclusion_edge, xlim[1], @@ -760,7 +861,7 @@ def plot_pure_eb_correlations( def plot_cosebis_scale_cut_heatmap( - cosebis_results, gg, version, output_path, fiducial_scale_cut=None + cosebis_results, edges, version, output_path, fiducial_scale_cut=None ): """ Create 2D heatmaps showing how COSEBIs statistics vary across different scale cuts. @@ -769,8 +870,8 @@ def plot_cosebis_scale_cut_heatmap( ---------- cosebis_results : dict Dictionary with scale cut tuples as keys, containing 'chi2_E' and 'pte_B' values - gg : treecorr.GGCorrelation - Correlation function object for bin edges + edges : tuple of numpy.ndarray + ``(left_edges, right_edges)`` of the grid the scale cuts index version : str Version string for main title output_path : str @@ -778,7 +879,8 @@ def plot_cosebis_scale_cut_heatmap( fiducial_scale_cut : tuple, optional (min_scale, max_scale) for cross-hatching """ - nbins = gg.nbins + left_edges, right_edges = edges + nbins = len(left_edges) # Initialize matrices snrs = [np.sqrt(result["chi2_E"]) for result in cosebis_results.values()] @@ -794,9 +896,9 @@ def plot_cosebis_scale_cut_heatmap( pte_matrix[row, column] = pte # Fill matrices from COSEBIs results - for i in range(len(gg.left_edges)): - for j in range(i, len(gg.right_edges)): - scale_cut = (gg.left_edges[i], gg.right_edges[j]) + for i in range(len(left_edges)): + for j in range(i, len(right_edges)): + scale_cut = (left_edges[i], right_edges[j]) result = cosebis_results.get(scale_cut) if result is not None: @@ -857,7 +959,9 @@ def plot_cosebis_scale_cut_heatmap( # Add fiducial scale cut cross-hatching if provided if fiducial_scale_cut is not None: min_scale, max_scale = fiducial_scale_cut - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges( + left_edges, right_edges, min_scale, max_scale + ) if stop_bin > start_bin and start_bin < nbins and stop_bin > 0: # Add cross-hatching at the fiducial scale cut matrix element @@ -884,10 +988,10 @@ def plot_cosebis_scale_cut_heatmap( # Set angular scale ticks tick_indices = np.arange(0, nbins) x_tick_labels = [ - f"{gg.left_edges[i]:.1f}" for i in tick_indices if i < len(gg.left_edges) + f"{left_edges[i]:.1f}" for i in tick_indices if i < len(left_edges) ] y_tick_labels = [ - f"{gg.right_edges[i]:.1f}" for i in tick_indices if i < len(gg.right_edges) + f"{right_edges[i]:.1f}" for i in tick_indices if i < len(right_edges) ] x_tick_positions = tick_indices + 0.5 y_tick_positions = tick_indices + 0.5 @@ -939,8 +1043,9 @@ def plot_pte_2d_heatmaps( fiducial_xim_scale_cut : tuple, optional (min_scale, max_scale) for xi- fiducial analysis, shown as cross-hatched """ - gg = results["gg"] - nbins = gg.nbins + theta = results["theta"] + edges = (results["left_edges"], results["right_edges"]) + nbins = len(theta) pte_xip_B = results["pte_matrices"]["xip_B"] pte_xim_B = results["pte_matrices"]["xim_B"] @@ -1002,7 +1107,7 @@ def plot_pte_2d_heatmaps( for ax_idx, fiducial_scale_cut in enumerate(fiducial_scale_cuts): if fiducial_scale_cut is not None: min_scale, max_scale = fiducial_scale_cut - start_bin, stop_bin = scale_cut_to_bins(gg, min_scale, max_scale) + start_bin, stop_bin = bins_from_edges(*edges, min_scale, max_scale) if stop_bin > start_bin and start_bin < nbins and stop_bin > 0: rect_x = start_bin @@ -1026,12 +1131,8 @@ def plot_pte_2d_heatmaps( # Set angular scale ticks tick_indices = np.arange(0, nbins) - x_tick_labels = [ - f"{gg.left_edges[i]:.1f}" for i in tick_indices if i < len(gg.left_edges) - ] - y_tick_labels = [ - f"{gg.right_edges[i]:.1f}" for i in tick_indices if i < len(gg.right_edges) - ] + x_tick_labels = [f"{edges[0][i]:.1f}" for i in tick_indices if i < len(edges[0])] + y_tick_labels = [f"{edges[1][i]:.1f}" for i in tick_indices if i < len(edges[1])] x_tick_positions = tick_indices + 0.5 y_tick_positions = tick_indices + 0.5 @@ -1171,10 +1272,8 @@ def save_pure_eb_results(results, output_path): output_path : str Output .npz file path """ - gg = results["gg"] - # Data vectors and covariance - save_dict = {"theta": gg.meanr, "cov": results["cov"]} + save_dict = {"theta": results["theta"], "cov": results["cov"]} for key in _EB_KEYS: save_dict[key] = results[key] @@ -1183,7 +1282,7 @@ def save_pure_eb_results(results, output_path): save_dict[f"pte_matrices_{key}"] = matrix # Metadata - save_dict["npatch"] = np.array(gg.npatch1) + save_dict["n_eff"] = np.array(results["n_eff"]) if "eb_samples" in results: save_dict["var_method"] = np.array("semi-analytic") save_dict["n_samples"] = np.array(results["eb_samples"].shape[0]) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 48251bbb..1d9fa196 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -563,18 +563,18 @@ def summarize_bmodes(self, fiducial_scale_cut=(12, 83), versions=None): # Pure E/B PTEs from stored results if ver in self._pure_eb_results: res = self._pure_eb_results[ver] - gg = res["gg"] + edges = (res["left_edges"], res["right_edges"]) try: for stat in ("xip_B", "xim_B", "combined"): row[stat] = _get_pte_from_scale_cut( - res["pte_matrices"][stat], gg, fiducial_scale_cut + res["pte_matrices"][stat], edges, fiducial_scale_cut ) except (KeyError, RuntimeError): pass cov_methods.add( "semi-analytic" if "eb_samples" in res - else f"jackknife ({gg.npatch1} patches)" + else f"jackknife ({res['n_eff']} patches)" ) # COSEBIs PTE from stored results diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index aa4657ba..312aadcd 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -271,7 +271,7 @@ def plot_cosebis( plot_cosebis_scale_cut_heatmap( results, - gg_temp, + (gg_temp.left_edges, gg_temp.right_edges), version, out_stub + "_scalecut_ptes.png", fiducial_scale_cut=fiducial_scale_cut, diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 71f5d76a..90157456 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -88,9 +88,13 @@ def calculate_pure_eb( - "xip_amb": Ambiguity mode for xi+. - "xim_amb": Ambiguity mode for xi-. - "cov": Covariance matrix for the pure E/B modes. - - "gg": The two-point correlation function object for the reporting binning. - - "gg_int": The two-point correlation function object for the - integration binning. + - "theta", "left_edges", "right_edges": Reporting-grid bin centres + and edges. + - "xip", "xim", "var_xip", "var_xim": Reporting-grid xi+/xi- and + their variances. + - "theta_int", "xip_int", "xim_int": Integration-grid xi+/xi-. + - "n_eff": Realisation count behind "cov" (jackknife patches or + MC draws), which sets the Hartlap debiasing. - "eb_samples": (only when using semi-analytical covariance) Semi-analytic EB samples used for covariance calculation. Shape: (n_samples, 6*nbins) @@ -269,19 +273,13 @@ def plot_pure_eb( ) # Calculate E/B statistics for all bin combinations - version_results = calculate_eb_statistics( - version_results, - cov_path_int=cov_path_int, - n_samples=n_samples, - **kwargs, - ) - - # Generate all plots using specialized plotting functions - gg, gg_int = version_results["gg"], version_results["gg_int"] + version_results = calculate_eb_statistics(version_results, **kwargs) # Integration vs Reporting comparison plot plot_integration_vs_reporting( - gg, gg_int, out_stub + "_integration_vs_reporting.png", version + version_results, + out_stub + "_integration_vs_reporting.png", + version, ) # E/B/Ambiguous correlation functions plot diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index e8e05c39..f51913fc 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -50,6 +50,7 @@ def _eb_inputs(): nbins=4, npatch=50 so the Hartlap factor (n_eff - nbins_eff - 2)/(n_eff-1) is well-defined and strictly positive for every scale-cut combination. + n_eff is the jackknife patch count, as it is for a jackknife covariance. The covariance is built SPD via A @ A.T + I; the B-mode vectors are O(1) so the chi-squared (and hence PTE) lands in a meaningful range rather than being saturated at 1.0. @@ -60,8 +61,13 @@ def _eb_inputs(): cov = A @ A.T + np.eye(6 * nbins) xip_B = rng.standard_normal(nbins) xim_B = rng.standard_normal(nbins) - gg = types.SimpleNamespace(nbins=nbins, npatch1=npatch) - return {"gg": gg, "cov": cov, "xip_B": xip_B, "xim_B": xim_B}, nbins + return { + "theta": np.geomspace(1.0, 100.0, nbins), + "n_eff": npatch, + "cov": cov, + "xip_B": xip_B, + "xim_B": xim_B, + }, nbins # --------------------------------------------------------------------------- @@ -229,8 +235,8 @@ def test_calculate_eb_statistics_pte_matrices(): """Pin representative PTE-matrix entries from the full 2D E/B analysis. Inputs are fixed (seed 12345, nbins=4, npatch=50, SPD cov = A@A.T + I, - O(1) B-mode vectors). With cov_path_int=None the Hartlap correction uses - n_eff = npatch = 50. For each of xip_B, xim_B and combined we pin the + O(1) B-mode vectors). The Hartlap correction uses n_eff = 50, the patch + count behind a jackknife covariance. For each of xip_B, xim_B and combined we pin the full-range entry [0, nbins-1] (start=0, stop=nbins) and an interior entry [0, 2] (start=0, stop=3). These chi2->sf PTE values are deterministic functions of the seeded input. @@ -240,7 +246,7 @@ def test_calculate_eb_statistics_pte_matrices(): arithmetic shifts them past tolerance. """ results, nbins = _eb_inputs() - out = b_modes.calculate_eb_statistics(results, cov_path_int=None) + out = b_modes.calculate_eb_statistics(results) pm = out["pte_matrices"] # Full-range entries [0, nbins-1]. @@ -271,13 +277,13 @@ def test_calculate_eb_statistics_has_teeth(): combined 0.99999 -> 0.0031. """ results, nbins = _eb_inputs() - out = b_modes.calculate_eb_statistics(results, cov_path_int=None) + out = b_modes.calculate_eb_statistics(results) pm = out["pte_matrices"] loud, _ = _eb_inputs() loud["xip_B"] = loud["xip_B"] * 10.0 loud["xim_B"] = loud["xim_B"] * 10.0 - out_loud = b_modes.calculate_eb_statistics(loud, cov_path_int=None) + out_loud = b_modes.calculate_eb_statistics(loud) pm_loud = out_loud["pte_matrices"] for key in ("xip_B", "xim_B", "combined"): @@ -285,3 +291,144 @@ def test_calculate_eb_statistics_has_teeth(): loud_pte = pm_loud[key][0, nbins - 1] assert loud_pte < quiet_pte assert loud_pte < 0.05 # louder B-modes are clearly rejected + + +# --------------------------------------------------------------------------- +# 6. Grid edges and the COSEBIs covariance seam +# --------------------------------------------------------------------------- + + +def test_log_bin_edges_matches_the_grid_stub(): + """Edges reconstructed from a binning are the ones TreeCorr would report. + + A part stores bin centres only, so a consumer rebuilds the edges from the + binning it was measured on; the two must agree bin for bin. + """ + left, right = b_modes.log_bin_edges(1.0, 100.0, _NBINS_GRID) + gg = _grid_gg() + npt.assert_allclose(left, gg.left_edges) + npt.assert_allclose(right, gg.right_edges) + # ...and they index scale cuts identically. + assert b_modes.bins_from_edges(left, right, 2.0, 50.0) == (2, 8) + + +def test_cosebis_scan_propagates_the_supplied_covariance(monkeypatch): + """The COSEBIs covariance is the ξ± covariance through the same kernel. + + The kernel is stubbed, so what is pinned is the seam: which ξ± covariance + sub-block is handed to the transform (the scale cut's, in [ξ+; ξ−] order) + and that Hartlap uses the supplied npatch. + """ + nbins, nmodes = _NBINS_GRID, 3 + theta = np.geomspace(1.2, 90.0, nbins) + cov_xipm = np.diag(np.arange(1.0, 2 * nbins + 1)) + seen = {} + + class _StubCOSEBIS: + def __init__(self, **kwargs): + seen["init"] = kwargs + + def cosebis_from_xipm(self, theta_cut, xip_cut, xim_cut, parallel=True): + seen["n_theta"] = len(theta_cut) + return np.ones(nmodes), np.full(nmodes, 2.0) + + def cosebis_covariance_from_xipm_covariance(self, theta_cut, cov_cut): + seen["cov_cut"] = cov_cut + return np.eye(2 * nmodes) + + module = types.ModuleType("cosmo_numba.B_modes.cosebis") + module.COSEBIS = _StubCOSEBIS + monkeypatch.setitem( + __import__("sys").modules, "cosmo_numba.B_modes.cosebis", module + ) + + left, right = b_modes.log_bin_edges(1.0, 100.0, nbins) + results = b_modes.cosebis_scan_from_xi( + theta, + np.arange(nbins) * 1e-5, + np.arange(nbins) * 2e-5, + cov_xipm, + left, + right, + nmodes=nmodes, + scale_cuts=[(2.0, 50.0)], + npatch=100, + ) + + (result,) = results.values() + # The cut is bins 2..8, so the covariance sub-block is those rows/cols in + # both the ξ+ and the ξ− half. + inds = np.concatenate([np.arange(2, 8), np.arange(2, 8) + nbins]) + npt.assert_array_equal(seen["cov_cut"], cov_xipm[np.ix_(inds, inds)]) + assert seen["n_theta"] == 6 + npt.assert_allclose(result["hartlap_factor"], (100 - 2 * nmodes - 2) / 99) + # χ² carries the Hartlap factor: modes are 1, cov is the identity. + npt.assert_allclose(result["chi2_E"], nmodes * result["hartlap_factor"]) + + +def test_cosebis_scan_theory_covariance_skips_hartlap(monkeypatch): + """A theory covariance has no realisations to debias, so Hartlap is 1.""" + nbins, nmodes = _NBINS_GRID, 2 + + class _StubCOSEBIS: + def __init__(self, **kwargs): + pass + + def cosebis_from_xipm(self, theta_cut, xip_cut, xim_cut, parallel=True): + return np.ones(nmodes), np.ones(nmodes) + + def cosebis_covariance_from_xipm_covariance(self, theta_cut, cov_cut): + return np.eye(2 * nmodes) + + module = types.ModuleType("cosmo_numba.B_modes.cosebis") + module.COSEBIS = _StubCOSEBIS + monkeypatch.setitem( + __import__("sys").modules, "cosmo_numba.B_modes.cosebis", module + ) + + left, right = b_modes.log_bin_edges(1.0, 100.0, nbins) + (result,) = b_modes.cosebis_scan_from_xi( + np.geomspace(1.2, 90.0, nbins), + np.zeros(nbins), + np.zeros(nbins), + np.eye(2 * nbins), + left, + right, + nmodes=nmodes, + npatch=None, + ).values() + assert result["hartlap_factor"] == 1 + + +def test_pure_eb_npz_carries_what_the_summary_reads(tmp_path): + """The .npz keys cv_summarize_bmodes reads are the ones the writer emits. + + The two live in different rules, so the contract between them — the PTE + matrices under ``pte_matrices_{stat}`` and the realisation count under + ``n_eff`` — is pinned here rather than discovered on a cluster run. + """ + results, nbins = _eb_inputs() + results.update( + {key: np.zeros(nbins) for key in b_modes._EB_KEYS if key not in results} + ) + results = b_modes.calculate_eb_statistics(results) + + out = tmp_path / "pure_eb_data.npz" + b_modes.save_pure_eb_results(results, str(out)) + saved = np.load(out) + + for stat in ("xip_B", "xim_B", "combined"): + assert f"pte_matrices_{stat}" in saved + assert saved[f"pte_matrices_{stat}"].shape == (nbins, nbins) + assert saved["n_eff"] == results["n_eff"] + npt.assert_allclose(saved["theta"], results["theta"]) + for key in b_modes._EB_KEYS: + assert key in saved + + # The summary reads the fiducial cut out of those matrices through the same + # helper the plots use, so a valid cut must resolve to a finite PTE. + edges = b_modes.log_bin_edges(1.0, 100.0, nbins) + pte = b_modes._get_pte_from_scale_cut( + saved["pte_matrices_xip_B"], edges, (1.0, 100.0) + ) + assert np.isfinite(pte) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index f50992d4..f25ed522 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -697,4 +697,4 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): # shape is pinned, not the values. cov = np.asarray(results["cov"]) assert cov.shape == (6 * nbins, 6 * nbins) - assert results["gg"].npatch1 == npatch + assert results["n_eff"] == npatch From d6d6f247b480a40bfd2a7249862bc438f182ddd5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 25 Sep 2026 15:23:39 +0200 Subject: [PATCH 27/83] Unpack cs_util's per-pair get_theo_xi in the pure-E/B MC covariance cs_util's get_theo_xi returns {pair: (xi+, xi-)}, so concatenating its return value raised TypeError and the semi-analytic pure-E/B covariance (pure_eb_covariance_mc, and the paper's precompute_pure_eb_chunk.py) could not run. One n(z) is one tracer pair: unpack that single entry, which also fails loudly if a tomographic n(z) is passed. The new test stubs the kernel and checks the draws centre on the binned theory mean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR --- .../scripts/precompute_pure_eb_chunk.py | 18 +++---- src/sp_validation/b_modes.py | 8 +-- src/sp_validation/tests/test_b_modes.py | 49 +++++++++++++++++++ 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/papers/bmodes/scripts/precompute_pure_eb_chunk.py b/papers/bmodes/scripts/precompute_pure_eb_chunk.py index 363a3192..99153096 100644 --- a/papers/bmodes/scripts/precompute_pure_eb_chunk.py +++ b/papers/bmodes/scripts/precompute_pure_eb_chunk.py @@ -127,15 +127,15 @@ def compute_chunk( row_sums = np.array(binning_matrix.sum(axis=1)).flatten() binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix - mean_int = np.concatenate( - get_theo_xi( - theta=theta_int, - z=z_dist[:, 0], - nz=z_dist[:, 1], - backend="ccl", - cosmo=cosmo_cov, - ) - ) + # One n(z) gives one tracer pair: get_theo_xi's single (xi+, xi-) entry. + (xi_pm,) = get_theo_xi( + theta=theta_int, + z=z_dist[:, 0], + nz=z_dist[:, 1], + backend="ccl", + cosmo=cosmo_cov, + ).values() + mean_int = np.concatenate(xi_pm) rng = np.random.default_rng(seed=42 + chunk_id) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index c4b9a2b7..7a3cd484 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -305,9 +305,11 @@ def pure_eb_covariance_mc( row_sums = np.array(binning_matrix.sum(axis=1)).flatten() binning_matrix = sparse.diags(1 / row_sums) @ binning_matrix - mean_int = np.concatenate( - get_theo_xi(theta=theta_int, z=z, nz=nz, backend="ccl", cosmo=cosmo) - ) + # One n(z) gives one tracer pair: get_theo_xi's single (xi+, xi-) entry. + (xi_pm,) = get_theo_xi( + theta=theta_int, z=z, nz=nz, backend="ccl", cosmo=cosmo + ).values() + mean_int = np.concatenate(xi_pm) samples_int = np.random.multivariate_normal(mean_int, cov_int, size=n_samples) samples_int_xip, samples_int_xim = ( samples_int[:, :nbins_int], diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index f51913fc..67c64376 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -432,3 +432,52 @@ def test_pure_eb_npz_carries_what_the_summary_reads(tmp_path): saved["pte_matrices_xip_B"], edges, (1.0, 100.0) ) assert np.isfinite(pte) + + +def test_pure_eb_covariance_mc_draws_around_the_theory_mean(monkeypatch): + """The MC draws centre on cs_util's theory ξ±, binned to the reporting grid. + + ``get_theo_xi`` returns ``{pair: (ξ+, ξ−)}``; one n(z) is one pair. With a + zero covariance every draw is the theory mean, so a stub kernel that echoes + its reporting-grid ξ± pins the unpack, the [ξ+; ξ−] order and the binning. + """ + nrep = 4 + left, right = b_modes.log_bin_edges(2.0, 50.0, nrep) + theta = np.sqrt(left * right) + theta_int = np.geomspace(1.0, 100.0, 40) + xip_th, xim_th = theta_int.copy(), 2 * theta_int + + monkeypatch.setattr( + b_modes, "get_theo_xi", lambda **kw: {"W0xW0": (xip_th, xim_th)} + ) + + def _echo(theta, theta_int, xip, xim, xip_int, xim_int, tmin, tmax, parallel): + zeros = np.zeros_like(xip) + return xip, xim, zeros, zeros, zeros, zeros + + module = types.ModuleType("cosmo_numba.B_modes.schneider2022") + module.get_pure_EB_modes = _echo + monkeypatch.setitem( + __import__("sys").modules, "cosmo_numba.B_modes.schneider2022", module + ) + + cov, eb_samples = b_modes.pure_eb_covariance_mc( + theta=theta, + left_edges=left, + right_edges=right, + theta_int=theta_int, + cov_int=np.zeros((2 * len(theta_int), 2 * len(theta_int))), + z=np.linspace(0.01, 2.0, 50), + nz=np.ones(50), + cosmo=None, + n_samples=3, + ) + + inside = [(theta_int >= lo) & (theta_int < hi) for lo, hi in zip(left, right)] + expected_xip = np.array([xip_th[m].mean() for m in inside]) + expected_xim = np.array([xim_th[m].mean() for m in inside]) + assert eb_samples.shape == (3, 6 * nrep) + for draw in eb_samples: + npt.assert_allclose(draw[:nrep], expected_xip) + npt.assert_allclose(draw[nrep : 2 * nrep], expected_xim) + npt.assert_allclose(cov, 0.0, atol=1e-20) From 3dc5eafc032fa6c75ba408d4b91e7dff72bab323 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Fri, 25 Sep 2026 16:38:31 +0200 Subject: [PATCH 28/83] cosmo_val: explicit catalogue config, forwarded constructor settings, patch_number from cat_config (#350) * xi, rho_tau_stats: pass catalogue config and output dir explicitly run_2pcf fell back to ./cat_config.yaml relative to the Snakemake cwd, and run_rho_tau chdir'd into a hardcoded pure_eb checkout. Both rules now pass CAT_CONFIG and COSMO_VAL, and the scripts require them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01KQ1cGfFTghPp3zw4KCYTQh * cv_init_params: forward catalogue config and every constructor default CosmologyValidation read ./cat_config.yaml from the rule's cwd and silently used constructor defaults for pseudo-Cl binning, noise bias, seeds and more. cv_init_params now passes CAT_CONFIG and forwards CV_INIT_KEYS from config['cosmo_val']; the paper config spells out the previous defaults, so behaviour is unchanged. test_cv_init_params fails if a new defaulted keyword is neither forwarded nor exempted with a reason. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01KQ1cGfFTghPp3zw4KCYTQh * rho/tau: read jackknife patch_number from the catalogue config get_params_rho_tau and set_params_rho_tau each hardcoded a survey-name table (120 for DES/SP_axel_v0.0, else 150) and ignored patch_number in cat_config.yaml. Every entry now declares patch_number (backfilled with the table's value) and both functions read it, failing loud when absent. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01KQ1cGfFTghPp3zw4KCYTQh * resolve_paths_for_version: skip non-mapping entry values Top-level scalars such as patch_number made the 'path' membership test raise TypeError on ints (and substring-match on strings). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01KQ1cGfFTghPp3zw4KCYTQh --------- Co-authored-by: Claude Fable 5.1 --- cosmo_val/cat_config.yaml | 27 ++++++++ papers/cosmo_val/config/config.yaml | 16 +++++ src/sp_validation/cosmo_val/core.py | 2 +- .../cosmo_val/psf_systematics.py | 14 +--- src/sp_validation/rho_tau.py | 19 ++--- .../tests/test_cv_init_params.py | 54 +++++++++++++++ src/sp_validation/tests/test_pseudo_cl.py | 1 + workflow/common.py | 69 ++++++++++++------- workflow/rules/twopoint.smk | 4 ++ workflow/scripts/cv_runner.py | 6 +- workflow/scripts/run_2pcf.py | 8 +-- workflow/scripts/run_rho_tau.py | 3 +- 12 files changed, 163 insertions(+), 60 deletions(-) create mode 100644 src/sp_validation/tests/test_cv_init_params.py diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 49cdfcc9..a4d34bee 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -38,6 +38,7 @@ DES: e1_col: obs_e1 e2_col: obs_e2 path: psf_y3a1-v29.fits + patch_number: 120 SP_axel_v0.0: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP @@ -77,6 +78,7 @@ SP_axel_v0.0: e1_col: HSM_G1_STAR e2_col: HSM_G2_STAR path: star_cat.fits + patch_number: 120 SP_v0.1.1: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP @@ -110,6 +112,7 @@ SP_v0.1.1: e1_col: HSM_G1_STAR e2_col: HSM_G2_STAR path: star_cat.fits + patch_number: 150 SP_v1.3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -145,6 +148,7 @@ SP_v1.3: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.0.3.fits + patch_number: 150 SP_v1.3.6: subdir: /n17data/UNIONS/WL/v1.3.x pipeline: SP @@ -188,6 +192,7 @@ SP_v1.3.6: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.0.3.fits + patch_number: 150 SP_v1.4.5: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -231,6 +236,7 @@ SP_v1.4.5: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.5.A: subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 pipeline: SP @@ -271,6 +277,7 @@ SP_v1.4.5.A: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.5_bright: subdir: /n17data/murray/unions_cats pipeline: SP @@ -311,6 +318,7 @@ SP_v1.4.5_bright: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.5_faint: subdir: /n17data/murray/unions_cats pipeline: SP @@ -351,6 +359,7 @@ SP_v1.4.5_faint: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6_glass_mock: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -393,6 +402,7 @@ SP_v1.4.6_glass_mock: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.5_intermediate: subdir: /n17data/murray/unions_cats pipeline: SP @@ -433,6 +443,7 @@ SP_v1.4.5_intermediate: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -476,6 +487,7 @@ SP_v1.4.6: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6.3: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -519,6 +531,7 @@ SP_v1.4.6.3: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6.3_B: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -562,6 +575,7 @@ SP_v1.4.6.3_B: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6.3_C: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -605,6 +619,7 @@ SP_v1.4.6.3_C: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6.3_A: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -648,6 +663,7 @@ SP_v1.4.6.3_A: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6_ecut07: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -691,6 +707,7 @@ SP_v1.4.6_ecut07: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6.1: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -780,6 +797,7 @@ SP_v1.4.7: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.8: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -823,6 +841,7 @@ SP_v1.4.8: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.11.2: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -866,6 +885,7 @@ SP_v1.4.11.2: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.11.3: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -910,6 +930,7 @@ SP_v1.4.11.3: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.12.3: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -955,6 +976,7 @@ SP_v1.4.12.3: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.13.3: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1000,6 +1022,7 @@ SP_v1.4.13.3: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.11.3_ecut07: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1043,6 +1066,7 @@ SP_v1.4.11.3_ecut07: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + patch_number: 150 SP_v1.4.6.3_uncal: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x @@ -1176,6 +1200,7 @@ SP_v1.4_LFmask_8k: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits + patch_number: 150 SP_v1.4_LFmask_8k_noalpha: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP @@ -1216,6 +1241,7 @@ SP_v1.4_LFmask_8k_noalpha: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits + patch_number: 150 nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz dndz: @@ -1267,3 +1293,4 @@ SP_v1.6.6: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.6.a.fits + patch_number: 150 diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 98409371..a0323b7c 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -30,6 +30,22 @@ cosmo_val: nrandom_cell: 100 cell_method: catalog nside_mask: 8192 + cov_estimate_method: th + compute_cov_rho: true + n_cov: 100 + var_method: jackknife + quantile: 0.1587 + ylim_xi_sys_ratio: [-0.02, 0.5] + fiducial_input_inka: coupled + # Pseudo-Cl map resolution, ell binning and noise/randoms settings + nside: 1024 + binning: powspace + power: 0.5 + n_ell_bins: 32 + ell_step: 10 + pol_factor: true + noise_bias_method: analytic + cell_seed: 8192 path_onecovariance: "/home/guerrini/OneCovariance/" rho_tau_method: lsq diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 1d9fa196..33113370 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -288,7 +288,7 @@ def resolve_paths_for_version(ver): """Resolve relative paths for a version using its subdir.""" subdir = Path(cc[ver]["subdir"]) for section in cc[ver].values(): - if "path" in section: + if isinstance(section, dict) and "path" in section: path = Path(section["path"]) section["path"] = ( str(path) if path.is_absolute() else str(subdir / path) diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 9f8a247e..98527490 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -98,17 +98,8 @@ def plot_tau_stats(self, plot_tau_m=False): + f"{os.path.abspath(self.tau_stat_handler.catalogs._output)}/{savefig}", ) - def set_params_rho_tau(self, params, params_psf, survey="other"): - params = {**params} - if survey in ("DES", "SP_axel_v0.0", "SP_axel_v0.0_repr"): - params["patch_number"] = 120 - print("DES, jackknife patch number = 120") - elif survey in ("SP_v1.4-P3", "SP_v1.4-P3_LFmask"): - params["patch_number"] = 120 - print("SP_v1.4, jackknife patch number =120") - else: - params["patch_number"] = 150 - + def set_params_rho_tau(self, params, params_psf, patch_number, survey="other"): + params = {**params, "patch_number": patch_number} params["ra_PSF_col"] = params_psf["ra_col"] params["dec_PSF_col"] = params_psf["dec_col"] params["e1_PSF_col"] = params_psf["e1_PSF_col"] @@ -149,6 +140,7 @@ def calculate_rho_tau_fits(self): params = self.set_params_rho_tau( self.results[ver]._params, self.cc[ver]["psf"], + self.cc[ver]["patch_number"], survey=ver, ) diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 6874a08c..62ae8bf8 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -17,21 +17,12 @@ def _extract_xip(correlations): def get_params_rho_tau(cat, survey="other"): + """Rho/tau parameters for one catalogue-config entry ``cat``. - # Set parameters - params = {} - # TODO to yaml file - if survey == "DES": - params["patch_number"] = 120 - print("DES, jackknife patch number = 120") - elif survey == "SP_axel_v0.0": - params["patch_number"] = 120 - print("SP_Axel_v0.0, jackknife patch number =120") - elif survey == "SP_v1.4-P3" or survey == "SP_v1.4-P3_LFmask": - params["patch_number"] = 120 - print("SP_v1.4, jackknife patch number =120") - else: - params["patch_number"] = 150 + The jackknife patch count is the entry's ``patch_number``; a missing key + raises ``KeyError``. + """ + params = {"patch_number": cat["patch_number"]} params["ra_PSF_col"] = cat["psf"]["ra_col"] params["dec_PSF_col"] = cat["psf"]["dec_col"] params["e1_PSF_col"] = cat["psf"]["e1_PSF_col"] diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py new file mode 100644 index 00000000..f769c179 --- /dev/null +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -0,0 +1,54 @@ +"""Guard: the workflow sets every CosmologyValidation constructor default. + +``workflow/common.py::cv_init_params`` builds the constructor kwargs for every +cosmo_val rule. A keyword it does not forward falls back to the constructor +default without any trace in the run config, so each keyword with a default is +either forwarded or exempted below with its reason. +""" + +import importlib.util +import inspect +from pathlib import Path + +import yaml + +from sp_validation.cosmo_val import CosmologyValidation + +REPO = Path(__file__).resolve().parents[3] + +EXEMPT = { + "output_dir": "rules set the output tree via the run directory / COSMO_VAL", + "blind": "None keeps the n(z) blind declared in the catalogue config", +} + + +def _load_common(): + spec = importlib.util.spec_from_file_location( + "workflow_common", REPO / "workflow" / "common.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _defaulted_keywords(): + params = inspect.signature(CosmologyValidation.__init__).parameters.values() + return {p.name for p in params if p.default is not inspect.Parameter.empty} + + +def test_every_default_is_forwarded_or_exempt(): + config = yaml.safe_load( + (REPO / "papers" / "cosmo_val" / "config" / "config.yaml").read_text() + ) + forwarded = set(_load_common().cv_init_params(config)) + unaccounted = _defaulted_keywords() - forwarded - set(EXEMPT) + assert not unaccounted, ( + f"CosmologyValidation defaults not forwarded by cv_init_params: " + f"{sorted(unaccounted)}. Forward them from config['cosmo_val'] or add " + f"them to EXEMPT with a reason." + ) + + +def test_exemptions_are_live_keywords(): + stale = set(EXEMPT) - _defaulted_keywords() + assert not stale, f"EXEMPT names keywords the constructor lacks: {sorted(stale)}" diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 45a2366d..2b5ccf51 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -146,6 +146,7 @@ def _write_synthetic_config(tmp_path): "shear": shear_cfg, "star": {**psf_cfg}, "psf": psf_cfg, + "patch_number": 150, }, } config_path = tmp_path / "config.yaml" diff --git a/workflow/common.py b/workflow/common.py index 506f7cb2..32a76b83 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -302,10 +302,10 @@ def get_shear_catalog(wildcards): # Sentinel directory for pure-plot leaf rules (no natural data-product output). CV_SENTINELS = COSMO_VAL / "snakemake_sentinels" -# Working directory in which `CosmologyValidation` must be instantiated: it -# reads `./cat_config.yaml` and writes to `./output` by default. Resolved to -# the live (non-worktree) checkout so rules find the catalog config and share -# the output tree with interactive runs. +# Working directory in which `CosmologyValidation` is instantiated: its +# catalogue config comes explicitly from CAT_CONFIG, and it writes to +# `./output` unless COSMO_VAL is set. Resolved to the live (non-worktree) +# checkout so rules share the output tree with interactive runs. CV_RUNDIR = "/n17data/cdaley/unions/code/sp_validation/cosmo_val" @@ -324,31 +324,52 @@ def cv_basename(version, fiducial=None): ) +# CosmologyValidation constructor kwargs read from config["cosmo_val"]. Every +# keyword with a default is either here or explicitly exempted in +# src/sp_validation/tests/test_cv_init_params.py, so no default applies silently. +CV_INIT_KEYS = ( + "rho_tau_method", + "cov_estimate_method", + "compute_cov_rho", + "n_cov", + "theta_min", + "theta_max", + "nbins", + "var_method", + "npatch", + "quantile", + "theta_min_plot", + "theta_max_plot", + "ylim_alpha", + "ylim_xi_sys_ratio", + "nside", + "nside_mask", + "binning", + "power", + "n_ell_bins", + "ell_step", + "pol_factor", + "cell_method", + "noise_bias_method", + "fiducial_input_inka", + "nrandom_cell", + "cell_seed", + "path_onecovariance", + "cosmo_params", +) + + def cv_init_params(config, version_list=None): """Assemble the CosmologyValidation(...) constructor kwargs from config. Centralizes the run-specific instantiation so every cosmo_val rule script - builds an identical `cv`. `version_list` overrides config["versions"] (used - by per-version rules that pass a single version). + builds an identical `cv`. The catalogue config is always CAT_CONFIG, never + the constructor's cwd-relative default. `version_list` overrides + config["versions"] (used by per-version rules that pass a single version). """ cv = config["cosmo_val"] - params = dict( + return dict( versions=version_list if version_list is not None else config["versions"], - npatch=cv["npatch"], - theta_min=cv["theta_min"], - theta_max=cv["theta_max"], - nbins=cv["nbins"], - theta_min_plot=cv["theta_min_plot"], - theta_max_plot=cv["theta_max_plot"], - ylim_alpha=cv["ylim_alpha"], - nrandom_cell=cv["nrandom_cell"], - cell_method=cv["cell_method"], - nside_mask=cv["nside_mask"], + catalog_config=CAT_CONFIG, + **{key: cv[key] for key in CV_INIT_KEYS}, ) - if cv.get("path_onecovariance"): - params["path_onecovariance"] = cv["path_onecovariance"] - if cv.get("rho_tau_method"): - params["rho_tau_method"] = cv["rho_tau_method"] - if cv.get("cosmo_params"): - params["cosmo_params"] = cv["cosmo_params"] - return params diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 275afc3c..b2df6c7c 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -15,6 +15,8 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + cat_config=CAT_CONFIG, + output_dir=str(COSMO_VAL), fits=False, resources: mem_mb=30000, @@ -35,6 +37,8 @@ rule rho_tau_stats: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + cat_config=CAT_CONFIG, + output_dir=str(COSMO_VAL), resources: mem_mb=30000, disk_mb=20000, diff --git a/workflow/scripts/cv_runner.py b/workflow/scripts/cv_runner.py index ba50847d..040cbe8a 100644 --- a/workflow/scripts/cv_runner.py +++ b/workflow/scripts/cv_runner.py @@ -29,9 +29,9 @@ def make_cv(snakemake): """Build a CosmologyValidation from a rule's ``snakemake.params``. ``params["cv_init"]`` is the kwargs dict assembled by common.cv_init_params. - The object is created with the run directory as cwd so it finds - ``cat_config.yaml`` and writes under ``output/`` exactly as interactive runs - do. + The catalogue config path arrives explicitly in ``cv_init``; the object is + created with the run directory as cwd so it writes under ``output/`` + exactly as interactive runs do. """ from sp_validation.cosmo_val import CosmologyValidation diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2e1ccabf..86dffe27 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -64,12 +64,8 @@ def _from_snakemake(smk): max_sep=float(p["max_sep"]), nbins=int(p["nbins"]), npatch=int(p["npatch"]), - # cat_config / output_dir were previously resolved via an os.chdir into - # the cosmo_val dir + the COSMO_VAL env var; expose them as optional - # params so the rule can pass them explicitly, falling back to the - # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. - cat_config=p.get("cat_config", "./cat_config.yaml"), - output_dir=p.get("output_dir", None), + cat_config=p["cat_config"], + output_dir=p["output_dir"], save_fits=True, ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index fbf62a3c..bc11bfc0 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -32,7 +32,6 @@ params = snakemake.params # type: ignore # %% -os.chdir("/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val") print("Starting CosmologyValidation") # Use parameters passed from Snakemake rule @@ -42,6 +41,8 @@ theta_max=float(params["max_sep"]), nbins=int(params["nbins"]), npatch=int(params["npatch"]), + catalog_config=params["cat_config"], + output_dir=params["output_dir"], ) cv.calculate_rho_tau_stats() From 4c16c28d7f69ce2542a563fe2efd429a002dc254 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Fri, 25 Sep 2026 17:01:25 +0200 Subject: [PATCH 29/83] cosmo_val + Snakemake migration to the SACC writers (PRD #241 row 4) (#251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sacc_io): SACC read/write for the standard data-product layout Add sp_validation.sacc_io: the writer/reader layer for the two-file SACC layout that becomes the package's standard data-product format. {version}.sacc analysis vector — NZ tracers, coarse xi+/-, pseudo-Cl (EE/BB/EB) with a shared BandpowerWindow, COSEBIs, pure E/B, rho/tau PSF diagnostics; one FullCovariance assembled block-diagonally (zero cross-blocks). {version}_xi_fine COSEBIs/pure-EB integration input — same NZ tracers, fine-grid xi+/-, DiagonalCovariance from TreeCorr varxip/varxim. Covariance order is point-insertion order (SACC preserves it bitwise through FITS). Writers insert in the canonical order — xi+ then xi-, Cl (ee, bb, eb), COSEBIs (all En then all Bn), pure E/B in _EB_KEYS order (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, matching b_modes.calculate_eb_statistics), rho, then tau — but readers never assume global order: every getter resolves indices through s.indices(dtype, tracers, **tags). assemble_covariance validates that blocks are contiguous, ascending and tile the data vector exactly, failing loud otherwise. Custom data types (pure E/B, rho, tau) all parse under sacc.parse_data_type_name. Tag filters are plain kwargs; the tags={...} form silently selects nothing and is never used. Test suite (test_sacc_io.py, all synthetic and fast): per-writer round-trips (arrays/tags/windows/NZ bitwise), covariance block alignment and zero cross-blocks, assemble_covariance failure modes, DiagonalCovariance round-trip, extract() sub-covariance alignment, a tomographic multi-pair case, reader/writer mirroring on a mixed file, and the end-to-end two-file layout. 20 passed. Co-Authored-By: Claude Opus * fix(sacc_io): enforce ascending grids; readers in insertion order Fresh-eyes review caught a correctness bug: readers re-sorted selections by theta/ell/n, but covariance blocks and bandpower windows stay in insertion order. On a non-ascending grid the reader output silently desynchronised from its covariance, and get_pseudo_cl returned sorted cl arrays against unsorted window columns — internally inconsistent within one return tuple. Fix by construction, not by sort: - Writers validate their grids. add_xi/add_pure_eb/add_rho/add_tau require strictly ascending theta; add_pseudo_cl requires strictly ascending ell_eff (add_cosebis is inherently safe — it enumerates the mode index). Out-of-order grids raise a loud ValueError naming the argument. - Readers drop the sort entirely and return in s.indices (insertion) order, so every getter is covariance- and window-aligned for ANY file, and ascending for canonical files. The _sorted_* helpers are replaced by plain insertion-order accessors (_mean/_tag). Also: - _pair normalises (i, j) -> sorted, so get_xi(s, (1, 0)) addresses the same symmetric shear-shear pair as (0, 1) instead of a silent empty read. - Module docstring documents the tomographic ξ covariance ordering: insertion is pair-major ([pair0 xip; pair0 xim; pair1 xip; …]), supplied to assemble_covariance as one contiguous block matching add_xi call order; type-major converters (DES 2pt-FITS) permute explicitly via s.indices. - extract() docstring states tracers takes SACC names, not integer bins. New tests (26 total, was 20): writers reject non-ascending theta/ell; (1, 0) == (0, 1) round-trip; a 3-pair tomographic ξ covariance assembled as one contiguous pair-major block with per-pair sub-blocks recovered via extract(); get_pseudo_cl window column j <-> ell_eff[j] via window_ind tags. Co-Authored-By: Claude Opus * One SACC file per catalogue version: fine xi rides as grid='fine' points The two-file split (analysis + {version}_xi_fine.sacc) was premised on a 10000-bin fine grid; the production operating point (Paper II B-modes) is 1000 bins, where a dense per-pair fine covariance block is ~32 MB and the CosmoCov integration-binning covariance — which feeds pure-E/B and COSEBIs error propagation — has a natural home as a BlockDiagonal block alongside the analysis blocks. Layout test replaced with the one-file end-to-end case (dense fine block, extract() sub-covariance alignment, zero cross-blocks) plus a varxi-diagonal fallback test. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019R5eiy11Lihkgn4MKufXSp * Word fine-block covariance source tool-agnostically (OneCovariance go-forward) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019R5eiy11Lihkgn4MKufXSp * feat(sacc_io): type=data|mock stamp with fail-closed load; merge + update_statistic PRD #241 §4 (Mocks vs data): save() requires type='data'|'mock' and stamps it into metadata; load() raises on type='data' files lacking the concealed=True blinding stamp, so skipping the blind can never silently expose real data. Mocks load freely. allow_unblinded=True is the loud escape hatch reserved for the blinding/unblinding tooling. merge() wraps sacc.concatenate_data_sets thinly: per-statistic files combine in order, shared tracers stored once, covariance block-diagonal (library-enforced all-or-none), metadata union with loud conflicts. update_statistic() is the value-only merge-back for the extract -> conceal -> merge blinding flow: matches each sub point by (data_type, tracers, tags) and overwrites the value, leaving order and covariance untouched. Tests: all four data/mock x concealed/unconcealed quadrants, the escape hatch, stamp validation, merge (points, covariance, metadata conflict, mixed-covariance failure) and update_statistic (values-only, unique-match). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C * Address review: rename grid values to reporting/integration; clarify comments - grid tag values: coarse -> "reporting", fine -> "integration" (descriptive, not relative); tag kept because both grids share data type and tracer pair, so the tag is the sole disambiguator. Swept module docstrings and tests. - Module docstring now spells out why insertion order is load-bearing: covariance row/column i refers to the i-th inserted data point. - new_sacc: comment that psf_stars rides in the same tracer list as the NZ tracers only because a Sacc has one flat tracer namespace (bookkeeping, not physics). - source_name/_pair helpers kept (4 and 8 call sites) with one-line justifications of the conventions they centralize. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C * Simplify sacc_io writers/readers and test builders - Factor the theta-tagged insertion loop into _add_theta_series (add_rho, add_tau, add_pure_eb); add_pure_eb zips PURE_TYPES.values() against its signature order, and PURE_KEYS is derived from PURE_TYPES instead of restating it. - Factor the (theta, plus, minus) read pattern into _get_pm (get_xi, get_rho, get_tau). - add_xi hoists the optional-tag None-filtering out of the point loop; extract and merge lose their throwaway mutable dicts; get_cosebis builds its scale-cut tags in one expression. - Tests: shared _add_xi default-ξ builder and _xi_block/_cl_block/ _cosebi_block canonical index-block helpers replace ~60 lines of copy-pasted setup; test_readers_on_mixed_file builds on _multi_statistic_sacc. Behaviour unchanged; 41/41 tests green in the container. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C * fix(sacc_io): fail loud on unmatched selections; merge/update guards Adversarial review (pass 2) findings: - Sacc.indices returns an EMPTY array (warning only) on an unmatched selection; every reader, extract(), and covariance-block selector now funnels through a shared _indices guard that raises instead — a typo'd tag or non-bitwise-identical float scale cut can no longer propagate empty arrays downstream (assemble_covariance previously died with an opaque IndexError on the same path). - get_cosebis(scale_cut=None) on a file carrying several scale cuts silently concatenated them; now raises and asks for an explicit cut. - update_statistic let two sub points claim the same target point (last-write-wins); now raises. - merge: the library keeps the FIRST input's tracer on a name clash with no equality check; shared tracers are now verified identical (z, nz) across inputs before concatenation. 7 regression tests; 48 total. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt * Rebuild PR4 on feat/sacc-2-sacc-io: drop vendored sacc_io, keep migration The branch previously carried a stale vendored copy of sacc_io.py / test_sacc_io.py from before PR2's review rounds. Rebuilt directly on feat/sacc-2-sacc-io (8bd38171) so the canonical module is inherited, bringing over the cosmo_val + Snakemake born-as-SACC migration work from the old tip unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt * Migrate to canonical sacc_io API: typed save, fail-closed load, grid rename Sweep the migration code onto PR2's canonical vocabulary and contracts: - grid='coarse'/'fine' -> 'reporting'/'integration' everywhere (writer calls, readers, tests), including the internal DAG intermediates: {version}_xi_coarse* -> _xi_reporting*, _xi_fine -> _xi_integration, the xi_coarse/xi_fine Snakemake output keys, CANONICAL part names, cv_xi_reporting_sacc, write_xi_integration_sacc. - save(s, path) -> save(s, path, type=...): 'data' at every production writer (the cosmo_val pipeline measures the real UNIONS catalogues; GLASS mocks do not flow through these writers), 'mock' for synthetic test fixtures. assemble_sacc propagates its parts' type stamp rather than hardcoding, so mock parts assemble into a mock analysis file. - load(path) -> fail-closed load: pipeline-internal readbacks of freshly written pre-blind data parts pass allow_unblinded=True (blinding is a downstream Smokescreen step); mock fixtures load freely. Readers raising on unmatched selections needed no call-site changes: every get_* reads a statistic guaranteed present in the file just written or assembled. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt * sacc_io: guard merge() against inconsistent theta grids Consistency follows tagging semantics: the grid tag declares which binning a set of points lives on, so all same-length theta arrays under one tag value must be bitwise identical (sacc never validates angles, and grids diverging at floating-point level choke CosmoSIS downstream). Different lengths within a tag group pass (scale-cut subsets); grids under different tag values are unconstrained (reporting vs integration differ by design). The rho/tau/pure-EB writers now tag their points grid="reporting" by default (overridable via grid=), so they join xi's consistency group and no untagged group appears in our own files. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk * sacc_io: extend merge guard to ell grids and bandpower windows; tag pseudo-Cl grid="reporting" The theta consistency guard generalizes to both angular domains: theta and ell points are grouped separately by grid tag value, and within a tag value all same-length grids must be bitwise identical. Two ell series sharing a grid must also carry equal bandpower windows (window ells and weight matrix); series without windows skip that check. add_pseudo_cl now stamps grid="reporting" on every point by default (overridable), joining the merge guard's consistency groups; since sacc's add_ell_cl accepts no extra tags, its per-point insertion (ell + shared window + window_ind) is inlined. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk * Store covariance block-diagonally in assemble_covariance assemble_covariance now passes sacc.add_covariance a list of per-block matrices instead of a dense zero-filled N×N array. sacc.BaseCovariance.make turns a list into a BlockDiagonalCovariance (one FITS table per block, Σ block² on disk), so cross-blocks are zero and implicit rather than materialized. Validation (contiguous/ascending indices, no gap/overlap, square blocks matching their index span) is unchanged. Every existing consumer already read through the polymorphic .dense property, so only the type assertions needed updating (FullCovariance -> BlockDiagonalCovariance). Added a test that merging two files that already carry a BlockDiagonalCovariance (assembled via assemble_covariance) stays block-diagonal through merge and a save/load round-trip. Updated the module docstring's storage-cost discussion accordingly. Co-Authored-By: Claude Fable 5 * Unify pure-EB integration-grid default to 1000 bins pure_eb.py's calculate_pure_eb/plot_pure_eb defaulted nbins_int=100; cosebis.py's calculate_cosebis already defaulted to 1000, and every production config (papers/bmodes, papers/cosmo_val fiducial/pure_eb blocks) already overrides to 1000. This aligns the function-signature default with what every caller actually uses; config plumbing is untouched, so any explicit override still wins. The papers/cosmo_val/config/config.yaml cosebis.nbins_int (currently 2000, production numerics) is deliberately left unchanged — see report. Co-Authored-By: Claude Fable 5 * sacc_io: make optional statistic components genuinely optional Writers no longer force components an analysis may not have computed: add_pseudo_cl's BB/EB, add_cosebis's Bn, and add_pure_eb's B/ambiguous blocks now default to None and are simply not written when omitted (add_pure_eb still requires each +/- pair together). Structural arguments (grids, tracer/bin identifiers, the value for a component you ARE adding) remain required with no default. Composite readers (get_pseudo_cl, get_cosebis, get_pure_eb) return None / omit the key for an absent optional component instead of raising, via a new _mean_optional helper; a selection naming a missing component explicitly (s.indices, _mean, extract) still fails loud, per the existing empty-selection guard. merge() and assemble_covariance work unchanged on files with only a subset of components. Documents the optionality contract in the module docstring, and adds partial-file round-trip, merge, and explicit-selection-fails-loud tests. Co-Authored-By: Claude Fable 5 * cosmo_val config: align COSEBIs integration grid with the 1000-bin default cosebis.nbins_int was 2000; every other integration-grid entry in this config (pure_eb, the fiducial block) is already 1000, and cosebis.py's own function defaults are 1000. Unify. Co-Authored-By: Claude Fable 5 * feat(sacc): one terminal file + fail-closed assembly Fold the integration-grid ξ± into the single terminal {version}.sacc, and make part loading fail closed on unblinded real data. Integration ξ± (grid='integration') is no longer its own terminal product. The xi_highres part is now gathered by rule assemble_sacc into {version}.sacc as tagged points, next to the reporting ξ± block. It is fiducial-only (the 10k-bin MPI run emits only the fiducial part), so it joins the fiducial version's terminal file alone. assemble_sacc.py adds xi_integration to CANONICAL; its own DiagonalCovariance passes straight through. assemble_sacc.py no longer loads every part with allow_unblinded=True. The run type (data|mock, from config, default data) gates it: mock runs load freely, data runs fail closed unless a part carries the concealed=True stamp. This is the seam for PR #253's blind-at-birth — a concealed data part then assembles with allow_unblinded=False untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * docs(workflow): mark glass-mock A/B/C 'blind' as distinct from Smokescreen The pseudo_cl / pseudo_cl_cov rules and the shared BLINDS list use a 'blind' wildcard that is the glass-mock multi-catalogue A/B/C variant, not Smokescreen blinding. Add prominent comments at the BLINDS definition, the wildcard constraint, and both rules so the two axes are not confused. A full rename is avoided: 'blind' is baked into on-disk filenames we do not own (external nz_{version}_{A|B|C}.txt) and into the covariance / inference path builders. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * fix(sacc): reconcile writers with the merged sacc_io API Resolve two silent semantic conflicts from the base merge (no textual conflict, but the tests broke). - add_cosebis reordered its params to (s, bins, En, scale_cut, Bn=None). cosebis_to_sacc still passed the old positional order; call it by keyword. - assemble_covariance now builds a BlockDiagonalCovariance (one FITS table per block, validated ordering), not a dense FullCovariance. Update the assembler docstrings, the smk comments, and the assembled-file test assertions to the block-diagonal type. The assembled .dense is unchanged (cross-blocks are zero either way); single-part writer covariances stay FullCovariance. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * chore(deps): restore develop's uv.lock SSOT; drop stale firecrown-era override machinery The branch had accidentally superseded #266's reproducible-deps model (uv.lock as source of truth, `uv sync --frozen` in the Dockerfile) with an older firecrown/Smokescreen-blinding scheme: uv.lock deleted, dependency resolution done via `uv pip install --overrides uv-overrides.txt`, and a patch script to make pip-installed firecrown importable without NumCosmo. Firecrown has been dropped from the project, so all of that goes. Restores develop's pyproject.toml, uv.lock, Dockerfile, and the deploy-image blinding-stack smoke test wholesale; removes uv-overrides.txt and scripts/patch_firecrown.py. Keeps one genuine PR4-driven change: tightens the sacc constraint to >=2.4,<3, since sacc_io.py (this branch) uses concatenate_data_sets/BlockDiagonalCovariance, both from sacc's 2.x rewrite (develop's lock already resolves sacc to 2.4; this just makes the pyproject floor honest). Drops the unused numexpr addition — no code in the tree imports it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * chore(ci): restore develop's lint.yml (PR #264 server-side ruff autofix) lost in stale merge * revert(sacc): drop xi_integration from terminal file per #247 ruling Keep the integration-grid ξ± as its own per-part intermediate ({version}_xi_integration.sacc) rather than folding it into the terminal {version}.sacc. Per the #247 ruling (comment 5033716753), the terminal file carries the analysis vector only; COSEBIs/pure-E/B consume the integration part directly, and Snakemake provenance covers its traceability. This keeps the terminal file at tens-of-MB scale. Removes xi_integration from CANONICAL, the cv_xi_integration_sacc helper, and the fiducial-gated assemble input. Keeps the fail-closed allow_unblinded gate, glass A/B/C comments, and dependency restorations from the prior rework. The integration part stays blinded at birth on data runs (per #253). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * chore: drop files resurrected from a stale base (deleted on develop) The "Rebuild PR4" base re-added the cosmo_inference/ analysis tree that develop removed in #236 (Clean up cosmo_inference folder); the develop merge kept them because a delete-vs-readd does not conflict. Remove the 80 resurrected files (notebooks, cosmosis_config .ini set, cosmo_inference scripts, get_chi2 notebooks, pipeline shells) to match develop. None is PR4 scope — PR4 is the SACC migration (workflow/ rules + scripts, sacc-related src + tests). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * chore(2pcf): default integration grid to 1000 bins The B-modes paper found no substantial 1k-vs-10k difference on the integration grid, and develop already unified COSEBIs/pure-EB to nbins_int=1000. Drop the run_2pcf_highres.py 10000-bin default to 1000 and reword the docstring/comments (the Asgari 10k figure becomes context, not the operative number). The grid is config-driven (nbins_int); the MPI path stays available but single-process is the default at 1000 bins. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * feat(cosmo_val): cv_cosebis/cv_pure_eb consume the ξ± SACC parts; per-version xi_highres COSEBIs and pure-E/B now derive their E-mode outputs from the born-as-SACC ξ± parts on disk instead of only the raw catalogue recompute. This makes born blinding automatic downstream: whatever the part carries (blinded on data runs in PR #253) flows into the E-modes with no blind-aware consumer code. b_modes: add the values-only seams cosebis_from_xi and pure_eb_from_xi (copied from the sacc-6b spur, de-blinded) — same cosmo_numba kernels as the raw path. cosebis_to_sacc_part / pure_eb_to_sacc_part: add en_override / eb_override so the E-mode En (COSEBIs) and the six pure-mode arrays (pure-E/B) written to the SACC part come from the consumed part; Bn and the covariance stay blind-invariant from the raw estimator run. No concealment machinery (that is PR #253's). cv_cosebis.py / cv_pure_eb.py: keep the raw plot_* run (Bn + jackknife covariance need the catalogue and are blind-invariant), then load the integration part (COSEBIs) / reporting + integration parts (pure-E/B), re-derive the E-modes through the seams, and override both the SACC part and the diagnostic npz. pure-E/B reads the reporting-grid bin edges from the raw reporting gg (SACC stores centers only). Unconditional and version-agnostic. xi_highres: per version (was fiducial-hardcoded); in-container single-process at the config-driven 1000-bin grid (the 10k-bin bare-host MPI path is unnecessary). Grid comes from a dedicated cosmo_val.integration block ([0.08, 300] @ 1000) so the one part serves both consumers (pure-E/B full range, COSEBIs scale-cuts to 0.9); decoupled from covariance.smk's FIDUCIAL grid. Shared twopoint.smk falls back to the fiducial integration grid for configs without a cosmo_val section (e.g. papers/bmodes). The raw .txt byproduct is left undeclared to avoid an AmbiguousRuleException with rule xi; nothing in the DAG consumes it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD * Restore cosmo_inference/ to develop state (stale-base residue reverted newer develop edits) * One binning-agnostic xi rule (grid + covariance are config, not a second rule) The reporting and integration ξ± measurements were two rules and two drivers for one computation. Collapse them: rule `xi` measures any binning, files are named by that binning (.txt + .sacc), and the grid label + covariance treatment are resolved from the wildcards via XI_GRIDS (reporting → no covariance, the ξ block arrives at assembly from CosmoCov; integration → DiagonalCovariance from TreeCorr varxip/varxim). Workflows with no cosmo_val block fall back to their fiducial grids; an unnamed binning measures as plain reporting. run_2pcf.py gains --grid/--covariance and drops run_2pcf_highres.py entirely (its hand-rolled catalogue loader, cat_config resolver and bare-host MPI path were a second implementation of CosmologyValidation.calculate_2pcf). The SACC part now stamps the npatch actually measured. calculate_2pcf writes patch results + covariance only for npatch > 1, so the fine grid no longer serialises a dense (2*nbins)^2 block that nothing reads. Renames {version}_xi_reporting_.sacc and {version}_xi_integration.sacc to {version}_xi_.sacc; consumers go through cv_xi_sacc(version, grid). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014hSQTC6WwTH1p9w4FuKJGz * Restore develop files wrongly deleted by stale-base cleanup (realspace paper, image-sims, mask configs, candide profile) * Restore develop content wrongly reverted by stale base (.gitignore uv.lock, CONTRIBUTING lint gate, calibration/catalog modules) * simplify: trim comments to Google style; dedupe helpers; drop dead knobs Comment/docstring pass over the PR-4 diff: cut PR/issue archaeology, restated defaults and "identical/bit-matching" claims (not true under blinding), keeping the load-bearing contracts (canonical part order, placeholder-cov consequence, scale_cut semantics, TreeCorr bin edges, CovTauTh k-major plus-folded layout). Code: - common.py gains base_version() and pseudo_cl_tag(); generate_cosmocov_ini imports both instead of copying build_redshift_path, fixing the drift where the cov_th lookup stripped only _leak_corr while the n(z) path also stripped _ecut{N}. cosmo_val.smk / inference.smk share the one pseudo-Cl tag. - run_cosmocov_chain.sh anchors the checkout on its own location; container, bind list and cosmocov binary come from the environment. - assemble_sacc: drop the unused --pseudo-cl-cov-hdu knob (the three named HDUs are always written; missing ones now raise). - assemble_analysis_sacc seeds tracers + metadata from parts[0], retiring _n_source_bins and the nz/metadata arguments. - Covariance mode derives from the grid: XI_GRIDS loses its "covariance" field, xi_grid_of returns just the label and compares binnings numerically, and run_2pcf loses its --covariance flag. - inference.smk: drop the dead inference_prep / inference_fiducial rules. Also restores bandpower_window_from_workspace (deleted in error by f14774b2) and develop's NON_PATH_KEYS overlay entries in test_config_paths_exist. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014hSQTC6WwTH1p9w4FuKJGz * cv_cosebis: derive Bn from the integration part; unpatched integration path in claims Both COSEBIs values now come from the (blindable) integration ξ± SACC part through the cosebis_from_xi seam; only the jackknife covariance still comes from the raw patched measurement, where the patches live. papers/bmodes' integration ξ± path said npatch=FIDUCIAL['npatch']; the integration grid (XI_GRIDS, twopoint.smk) is unpatched, so pin it to 1. * test: make the COSEBIs override stub work under the blinded-part writer too * Leave inference.smk untouched: it is #255's to rewrite The dormant inference subsystem was being half-edited here (inference_prep deleted, comments trimmed, pseudo_cl_tag adopted) while #255 rewrites it wholesale. Restored to its merge-base-with-develop state; both paper workflows still dry-run. * Assemble against the real CosmoCov ξ± covariance; drop the placeholder The ξ± block now comes from the CosmoCov-processed covariance on the reporting binning, wired as a DAG input of assemble_sacc (cv_xi_cov), so requesting the terminal file builds the covariance chain. With a real block always present, the allow_placeholder_cov switch and the diagonal stand-in are gone: a missing covariance is a missing Snakemake input, and a cov-less part with nothing to inject still raises. The tests use a CosmoCov-format .txt fixture written with the same writer the seam reads. * covariance_process: run the launched checkout's cosmocov_process.py The rule shelled out to an absolute path through the deprecated pure_eb symlink, i.e. a different checkout on an unrelated branch — now on the critical path, since assemble_sacc depends on this rule. REPO_ROOT (common.py) anchors on the module's realpath, the same way the rest of the workflow resolves the running checkout under `module` composition. * Leave cosmo_inference/README.md untouched (inference is out of scope here) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM * Restore image_sims include + README section (out of scope here) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM * README: state the 3.12 requirement, not its rationale Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NmGA86b7YyQn54JFj78ssM * Comment sweep: each concept once, at the thing itself Kills repeats (the same grid/naming/canonical-order explanation restated at producer, rule and script) and comments that narrated what downstream code does — consumer names, rule wiring, pipeline shape — which rot as soon as the consumer moves. What stays is each thing's own contract plus the invariants a reader would violate: the inclusive scale-cut selection, the strict sub-range the pure-E/B integration grid must satisfy, why the integration grid carries no covariance, why an already-written part may be read back unblinded, and the covariance ordering assembly depends on. Also drops the stale rename note in generate_pseudo_cl (parts are born at their final path) and the duplicated REPO_ROOT anchoring rationale in common.py. * Each grid carries its own covariance; COSEBIs derives from ξ± arrays A grid is now a binning plus how its covariance is estimated, so one TreeCorr run per grid produces a part complete enough to work from: the dense jackknife block where there are patches, the varxip/varxim diagonal, or nothing. The COSEBIs scan gains an arrays-and-covariance seam (cosebis_scan_from_xi): the COSEBIs covariance was already the ξ± covariance through the same linear kernel as the modes, so a caller holding a data vector needs no estimator re-run — only the realisation count behind the covariance, for Hartlap. calculate_cosebis delegates to it, and the scale-cut heatmap takes bin edges rather than a correlation object; log_bin_edges reconstructs those edges from a binning, which is what a part-based consumer has. cv_cosebis becomes such a consumer: one part in, and the SACC part, the .npz scan and the figures all out of those values. The en/bn_override machinery it needed while values and covariance came from different runs goes with it. * cv_cosebis: bind the COSEBIs grid's part, and declare the figures The rule's inputs are now one ξ± part instead of a TreeCorr .txt plus the integration part, and the three companion figures are declared outputs rather than byproducts landing beside them. The COSEBIs grid (0.9–300 at 1000 bins, patched) is a row in the grid table, so the same xi rule measures it. * Pure E/B derives from its parts; per-patch vectors are never written The pure-E/B jackknife needed the patched correlation objects, which is the one thing a part cannot carry — and per Cail's ruling jackknife and blinding are a deprecated pair for derived statistics. So the covariance now comes from the Monte Carlo path calculate_pure_eb already had, extracted as pure_eb_covariance_mc: draws from the CosmoCov gaussian covariance on the integration grid, around a theory mean, through the same kernel as the modes. It depends on the covariance model and the grids, never on the measured vector, so it is blind-invariant by construction. cv_pure_eb consumes two parts plus that one covariance file and touches no catalogue. The results dict is now self-describing — the grids travel with the modes (theta, edges, the raw ξ± and their variances, n_eff) instead of a TreeCorr object riding along — so the statistics, plots and npz all work from values. That deletes the FakeGG stub the paper PTE script needed, and turns calculate_eb_statistics into a function of its results alone. calculate_2pcf no longer writes patch results into the .txt dump: a per-patch ξ± realisation is an unblinded data vector, nothing reads one back, and the covariance a consumer needs is the matrix the part carries. * cv_summarize_bmodes reads the products instead of recomputing them It re-ran plot_pure_eb / plot_cosebis / plot_pseudo_cl in-process to repopulate the in-memory result dicts, which meant the terminal diagnostic reached the catalogue — and rewrote the two consumers' figures from unblinded values on the way past. It now reads what those rules wrote: the pure-E/B PTE matrices and the COSEBIs B-mode PTE from their .npz products, and the pseudo-Cℓ BB spectrum from its part against the NaMaster covariance, both declared inputs. The summary can no longer disagree with the products it summarises. The table itself moves to print_bmode_summary, so the in-memory path (summarize_bmodes, for notebooks) and the file path print the same thing. A test pins the .npz key contract the two rules meet on. * Review round: one pseudo-Cl producer, analytic covariance wins, grid tags canonical Six findings from the review. One pseudo-Cl producer. The untagged diagnostic part and its rule are gone; the summary, the terminal file and now the figures all read the analysis part with its matching NaMaster covariance, so the three cannot disagree. The figures become a plot-only ingest rule like the other B-mode ones, over a plot_pseudo_cl_spectrum that draws one spectrum for every version — the three near-identical EE/EB/BB blocks were one function all along. The analytic covariance wins, loudly. A part whose analysis covariance is external (ξ± reporting, pseudo-Cl) now always takes the supplied block, which replaces the estimate it was born with rather than losing a race with it, and says so on stdout; a missing injection raises instead of silently keeping the part's own. The reporting part keeps carrying its jackknife — it is a real diagnostic, just not the analysis covariance. Grid tags are canonical. xi_binning stamped raw YAML (maxsep=300) while the measurement wrote float-normalised names (maxsep=300.0), so producer and consumer would have asked for different paths on the first real run. The table coerces once, where it is built. It moved to common.py to be testable at all, and the tests pin the round trip a filename makes. Also: cosebis_from_xi deleted (no callers, and its centre-based cut diverged from the edge-based one in use); the paper PTE script's unread cov_integration and npatch dropped, with the rule docstring corrected to say plainly that nothing there varies by blind; and the header DAG comment rewritten to the SACC-part graph it actually describes. * chore: delete dead cosmocov drivers and repoint dangling references `workflow/scripts/run_cosmocov_chain.sh` invoked `cosmo_inference/scripts/cosmocov_process.py`, which was deleted on develop (#236) and relocated as the snakemake-only `workflow/scripts/cosmocov_process.py` (driven by `workflow/rules/covariance.smk`). The chain script's only caller was `papers/bmodes/scripts/run_cov_sweep.sh`, whose own only caller was a human, so both are dead end-to-end and are removed. Tree sweep for the same class of rot, repointing what survives: - README: drop the firecrown/Smokescreen override paragraph. Both files it names (`uv-overrides.txt`, `scripts/patch_firecrown.py`) were removed in f5eb417 when firecrown was dropped, and there is no `blinding` extra left in pyproject. - CLAUDE.md / CONTRIBUTING.md: the single-test example named `tests/test_cosmology.py`, which does not exist; use `tests/test_cosmo_val.py`. - CLAUDE.md: the cosmo_inference section documented a `./pipeline.sh` driver with flags. There is no such script; the pipeline is Snakemake-orchestrated, so the section now carries the invocation from `cosmo_inference/README.md`. - `papers/bmodes/config/ecut_spec.md`: `workflow/config/config.yaml` and `workflow/rules/claims.smk` moved under `papers/bmodes/`; the catalog config is `cosmo_val/cat_config.yaml`, not `code/sp_validation/cosmo_val/…`. - `papers/bmodes/scripts/update_survey_stats.py`: usage docstring still gave its pre-move `workflow/scripts/` path. Co-Authored-By: Claude Fable 5.1 * papers Snakefiles: read cat_config through code/, not the deprecated pure_eb symlink Same file as common.py's CAT_CONFIG; the symlink is slated for removal. Co-Authored-By: Claude Fable 5.1 * papers/bmodes: read the pseudo-Cl spectrum from its SACC part The generic pseudo_cl rule writes a SACC part, so the paper's C_ell consumers read that instead of the FITS PSEUDO_CELL table: one shared reader (pseudo_cl_io.load_pseudo_cl_data over sacc_io.get_pseudo_cl) for the six scripts, and the claims.smk path helper points at the .sacc part. The bandpower covariance still comes from pseudo_cl_cov's FITS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR * Drop generate_cosmocov_ini.py and the README install section generate_cosmocov_ini.py was deleted on develop (#236) and came back with this branch's stale-base rebuild; nothing calls it. The README "Local Installation" section repeats docs/source/installation.rst and is not part of the SACC migration. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HeMhD6yCyrgBz5oz87bCtR --------- Co-authored-by: Claude Opus --- papers/bmodes/config/cl.md | 2 +- papers/bmodes/rules/claims.smk | 6 +- .../bb_covariance_blind_independence.py | 13 +- papers/bmodes/scripts/cl_data_vector.py | 23 +- .../bmodes/scripts/cl_version_comparison.py | 22 +- .../harmonic_config_cosebis_comparison.py | 22 +- .../scripts/harmonic_space_pte_matrices.py | 16 +- .../scripts/plot_cosebis_filter_overlay.py | 8 +- papers/bmodes/scripts/pseudo_cl_io.py | 19 + papers/bmodes/scripts/run_xi_sweep.py | 6 +- papers/cosmo_val/config/config.yaml | 10 +- pyproject.toml | 3 +- src/sp_validation/cosmo_val/core.py | 86 ++-- src/sp_validation/cosmo_val/pseudo_cl.py | 326 +++++---------- .../cosmo_val/psf_systematics.py | 33 ++ src/sp_validation/cosmo_val/real_space.py | 85 +--- src/sp_validation/cosmo_val/sacc_writers.py | 237 +++++++++++ src/sp_validation/pseudo_cl.py | 29 ++ src/sp_validation/tests/test_assemble_sacc.py | 300 ++++++++++++++ .../tests/test_bmodes_workflow_dry_run.py | 58 ++- src/sp_validation/tests/test_cli_seams.py | 57 +++ src/sp_validation/tests/test_pseudo_cl.py | 41 +- src/sp_validation/tests/test_sacc_writers.py | 389 ++++++++++++++++++ src/sp_validation/tests/test_xi_grids.py | 109 +++++ workflow/common.py | 119 +++++- workflow/rules/cosmo_val.smk | 337 +++++++++++---- workflow/rules/twopoint.smk | 46 ++- workflow/scripts/assemble_sacc.py | 203 +++++++++ workflow/scripts/cv_cosebis.py | 73 +++- workflow/scripts/cv_plot_pseudo_cl.py | 39 ++ workflow/scripts/cv_pseudo_cl.py | 13 - workflow/scripts/cv_pure_eb.py | 119 +++++- workflow/scripts/cv_summarize_bmodes.py | 90 ++-- workflow/scripts/generate_pseudo_cl.py | 65 ++- workflow/scripts/run_2pcf.py | 86 +++- workflow/scripts/run_rho_tau.py | 5 +- 36 files changed, 2453 insertions(+), 642 deletions(-) create mode 100644 papers/bmodes/scripts/pseudo_cl_io.py create mode 100644 src/sp_validation/cosmo_val/sacc_writers.py create mode 100644 src/sp_validation/tests/test_assemble_sacc.py create mode 100644 src/sp_validation/tests/test_cli_seams.py create mode 100644 src/sp_validation/tests/test_sacc_writers.py create mode 100644 src/sp_validation/tests/test_xi_grids.py create mode 100644 workflow/scripts/assemble_sacc.py create mode 100644 workflow/scripts/cv_plot_pseudo_cl.py delete mode 100644 workflow/scripts/cv_pseudo_cl.py diff --git a/papers/bmodes/config/cl.md b/papers/bmodes/config/cl.md index d178ddee..ad0b7d70 100644 --- a/papers/bmodes/config/cl.md +++ b/papers/bmodes/config/cl.md @@ -25,7 +25,7 @@ Pseudo-Cl estimation accounts for: ## Data Source Pseudo-Cl files generated by workflow rules using NaMaster: -- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Power spectrum estimates +- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc` — Power spectrum estimates - `pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix Location: `{COSMO_VAL_OUTPUT}/` (defined in Snakefile, typically `/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/`) diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/claims.smk index 3846b9bc..85891b25 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/claims.smk @@ -86,10 +86,10 @@ def _xi_reporting_path(version): def _xi_integration_path(version): - """Path to fine-binned 2PCF integration file.""" + """Path to fine-binned 2PCF integration file. Unpatched: values only, no covariance.""" return ( f"{COSMO_VAL_OUTPUT}/{version}_xi_minsep={FIDUCIAL['min_sep_int']}" - f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch={FIDUCIAL['npatch']}.txt" + f"_maxsep={FIDUCIAL['max_sep_int']}_nbins={FIDUCIAL['nbins_int']}_npatch=1.txt" ) @@ -121,7 +121,7 @@ def _pseudo_cl_path(version, blind="A", nbins=32): All leak-corrected versions use consistent local naming with blind and binning. """ - return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{COSMO_VAL_OUTPUT}/pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov_path(version, blind="A", nbins=32): diff --git a/papers/bmodes/scripts/bb_covariance_blind_independence.py b/papers/bmodes/scripts/bb_covariance_blind_independence.py index 6d38f750..256a79b1 100644 --- a/papers/bmodes/scripts/bb_covariance_blind_independence.py +++ b/papers/bmodes/scripts/bb_covariance_blind_independence.py @@ -22,6 +22,7 @@ import yaml from astropy.io import fits from plotting_utils import PAPER_MPLSTYLE +from pseudo_cl_io import load_pseudo_cl_data from sp_validation.b_modes import calculate_cosebis @@ -436,9 +437,8 @@ def main( nbins_int, ) - # Read ell bin centers from pseudo-Cl data file - with fits.open(pseudo_cl_path) as hdu: - ell_eff = hdu["PSEUDO_CELL"].data["ELL"] + # Read ell bin centers from the pseudo-Cl SACC part + ell_eff = load_pseudo_cl_data(pseudo_cl_path)["ELL"] # Compute ratios relative to blind A pure_eb_results = {} @@ -666,7 +666,10 @@ def _from_cli(argv=None): ap.add_argument( "--cosmo-val-dir", required=True, - help="COSMO_VAL output dir (pseudo_cl / pseudo_cl_cov FITS + xi_integration txt)", + help=( + "COSMO_VAL output dir (pseudo_cl SACC parts, pseudo_cl_cov FITS " + "+ xi_integration txt)" + ), ) ap.add_argument( "--covariance-dir", @@ -711,7 +714,7 @@ def _from_cli(argv=None): f"_nbins={nbins_int}_npatch={npatch}.txt", ) pseudo_cl_path = os.path.join( - a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.fits" + a.cosmo_val_dir, f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" ) out_dir = Path(a.out) diff --git a/papers/bmodes/scripts/cl_data_vector.py b/papers/bmodes/scripts/cl_data_vector.py index 6ee5a93f..4eeca960 100644 --- a/papers/bmodes/scripts/cl_data_vector.py +++ b/papers/bmodes/scripts/cl_data_vector.py @@ -26,6 +26,7 @@ get_powspace_bin_edges, iter_version_figures, ) +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) @@ -44,10 +45,8 @@ def _compute_pte_with_cuts(data, covariance, ell, ell_min, ell_max): def _load_pseudo_cl_data(pseudo_cl_path, pseudo_cl_cov_path): - """Load pseudo-Cl data and covariance from FITS files.""" - hdu = fits.open(pseudo_cl_path) - data = hdu["PSEUDO_CELL"].data - hdu.close() + """Load pseudo-Cl data from SACC and covariance from FITS.""" + data = load_pseudo_cl_data(pseudo_cl_path) ell = data["ELL"] cl_eb = data["EB"] @@ -163,7 +162,7 @@ def _create_cl_figure( # so the version sweep is self-contained (lc produces only the fiducial version). # --------------------------------------------------------------------------- def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): @@ -184,10 +183,9 @@ def _resolve_pseudo_cl_paths( """Resolve the pseudo-Cl and covariance paths for one version in the sweep. Every version is read from the reconstructed COSMO_VAL pattern, except the - fiducial version when explicit override paths are supplied: the lc-produced - fiducial FITS files are named `pseudo_cl_{version}.fits` / - `pseudo_cl_cov_{version}.fits` (no blind=/nbins= tokens), so pattern - reconstruction can't find them and an explicit path is required instead. + fiducial version when explicit override paths are supplied: lc-produced + fiducial SACC spectrum and FITS covariance files have untagged names, so + pattern reconstruction can't find them and explicit paths are required. """ if ( ver == fiducial_version @@ -333,7 +331,10 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version pseudo_cl_* / pseudo_cl_cov_* FITS", + help=( + "COSMO_VAL output dir with per-version pseudo_cl_*.sacc and " + "pseudo_cl_cov_*.fits" + ), ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( @@ -349,7 +350,7 @@ def _from_cli(argv=None): "--fiducial-pseudo-cl-path", default=None, help=( - "Explicit path to the fiducial version's pseudo-Cl FITS (lc output). " + "Explicit path to the fiducial version's pseudo-Cl SACC part (lc output). " "Requires --fiducial-version and --fiducial-pseudo-cl-cov-path." ), ) diff --git a/papers/bmodes/scripts/cl_version_comparison.py b/papers/bmodes/scripts/cl_version_comparison.py index cd283f2a..f70a6977 100644 --- a/papers/bmodes/scripts/cl_version_comparison.py +++ b/papers/bmodes/scripts/cl_version_comparison.py @@ -29,12 +29,13 @@ get_version_alpha, version_label, ) +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): @@ -55,10 +56,9 @@ def _resolve_pseudo_cl_paths( """Resolve the pseudo-Cl and covariance paths for one version in the sweep. Every version is read from the reconstructed COSMO_VAL pattern, except the - fiducial version when explicit override paths are supplied: the lc-produced - fiducial FITS files are named `pseudo_cl_{version}.fits` / - `pseudo_cl_cov_{version}.fits` (no blind=/nbins= tokens), so pattern - reconstruction can't find them and an explicit path is required instead. + fiducial version when explicit override paths are supplied: lc-produced + fiducial SACC spectrum and FITS covariance files have untagged names, so + pattern reconstruction can't find them and explicit paths are required. """ if ( ver == fiducial_version @@ -131,10 +131,7 @@ def main( fiducial_pseudo_cl_cov_path, ) - hdu = fits.open(pseudo_cl_path) - data = hdu["PSEUDO_CELL"].data - hdu.close() - + data = load_pseudo_cl_data(pseudo_cl_path) ell = data["ELL"] cl_bb = data["BB"] cl_eb = data["EB"] @@ -379,7 +376,10 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version pseudo_cl_* / pseudo_cl_cov_* FITS", + help=( + "COSMO_VAL output dir with per-version pseudo_cl_*.sacc and " + "pseudo_cl_cov_*.fits" + ), ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( @@ -395,7 +395,7 @@ def _from_cli(argv=None): "--fiducial-pseudo-cl-path", default=None, help=( - "Explicit path to the fiducial version's pseudo-Cl FITS (lc output). " + "Explicit path to the fiducial version's pseudo-Cl SACC part (lc output). " "Requires --fiducial-version and --fiducial-pseudo-cl-cov-path." ), ) diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index f8025a0e..dd34db22 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -32,6 +32,7 @@ iter_version_figures, version_label, ) +from pseudo_cl_io import load_pseudo_cl_data from sp_validation.b_modes import calculate_cosebis @@ -107,11 +108,10 @@ def _compute_harmonic_cosebis( pseudo_cl_path, pseudo_cov_path, nmodes, theta_min, theta_max ): """Compute COSEBIS from pseudo-C_ell and propagate covariance.""" - with fits.open(pseudo_cl_path) as hdul: - data = hdul["PSEUDO_CELL"].data - ell = np.asarray(data["ELL"], dtype=float) - cl_ee = np.asarray(data["EE"], dtype=float) - cl_bb = np.asarray(data["BB"], dtype=float) + data = load_pseudo_cl_data(pseudo_cl_path) + ell = np.asarray(data["ELL"], dtype=float) + cl_ee = np.asarray(data["EE"], dtype=float) + cl_bb = np.asarray(data["BB"], dtype=float) cosebis_obj = COSEBIS(theta_min, theta_max, nmodes) @@ -857,7 +857,10 @@ def _from_cli(argv=None): ap.add_argument( "--cosmo-val-dir", required=True, - help="COSMO_VAL output dir (pseudo_cl / pseudo_cl_cov FITS + xi_integration txt)", + help=( + "COSMO_VAL output dir (pseudo_cl SACC parts, pseudo_cl_cov FITS " + "+ xi_integration txt)" + ), ) ap.add_argument( "--covariance-dir", @@ -883,8 +886,9 @@ def _from_cli(argv=None): "--fiducial-pseudo-cl-path", default=None, help=( - "Explicit path to the fiducial 96-bin pseudo-Cl FITS reproduced by lc " - "(from lc's cl_bandpowers_fine; e.g. pseudo_cl_SP_v1.4.6.3_leak_corr.fits), " + "Explicit path to the fiducial 96-bin pseudo-Cl SACC part reproduced " + "by lc (from lc's cl_bandpowers_fine; e.g. " + "pseudo_cl_SP_v1.4.6.3_leak_corr.sacc), " "overriding the --cosmo-val-dir pattern lookup for --fiducial-version." ), ) @@ -934,7 +938,7 @@ def _from_cli(argv=None): if is_fiducial and a.fiducial_pseudo_cl_path else os.path.join( a.cosmo_val_dir, - f"pseudo_cl_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.fits", + f"pseudo_cl_{ver}_blind={a.blind}_powspace_nbins={cosebis_nbins}.sacc", ) ) # 96-bin pseudo-Cl covariance is intentionally NOT lc-repointed: lc did not diff --git a/papers/bmodes/scripts/harmonic_space_pte_matrices.py b/papers/bmodes/scripts/harmonic_space_pte_matrices.py index f52e1bc8..aeb031c6 100644 --- a/papers/bmodes/scripts/harmonic_space_pte_matrices.py +++ b/papers/bmodes/scripts/harmonic_space_pte_matrices.py @@ -26,12 +26,13 @@ make_pte_colormap, make_pte_norm, ) +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) def _pseudo_cl(results_dir, ver, blind="A", nbins=32): - return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.fits" + return f"{results_dir}/pseudo_cl_{ver}_blind={blind}_powspace_nbins={nbins}.sacc" def _pseudo_cl_cov(results_dir, ver, blind="A", nbins=32): @@ -48,7 +49,7 @@ def compute_pte_matrix( Parameters ---------- pseudo_cl_path : str - Path to pseudo-Cl FITS file. + Path to pseudo-Cl SACC part. pseudo_cl_cov_path : str Path to pseudo-Cl covariance FITS file. fiducial_ell_min : float, optional @@ -66,10 +67,7 @@ def compute_pte_matrix( Summary statistics. """ # Load pseudo-Cl data - hdu = fits.open(pseudo_cl_path) - data = hdu["PSEUDO_CELL"].data - hdu.close() - + data = load_pseudo_cl_data(pseudo_cl_path) ell = data["ELL"] cl_bb = data["BB"] n_ell = len(ell) @@ -533,7 +531,9 @@ def _from_cli(argv=None): ap.add_argument( "--results-dir", required=True, - help="COSMO_VAL output dir with per-version pseudo_cl_* / pseudo_cl_cov_* FITS", + help=( + "COSMO_VAL output dir with pseudo_cl_*.sacc parts and pseudo_cl_cov_*.fits" + ), ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") ap.add_argument( @@ -544,7 +544,7 @@ def _from_cli(argv=None): ap.add_argument( "--fiducial-pseudo-cl-path", default=None, - help="Explicit path to fiducial pseudo-Cl FITS produced by lc " + help="Explicit path to fiducial pseudo-Cl SACC part produced by lc " "(overrides pattern reconstruction for --fiducial-version)", ) ap.add_argument( diff --git a/papers/bmodes/scripts/plot_cosebis_filter_overlay.py b/papers/bmodes/scripts/plot_cosebis_filter_overlay.py index 8de785b3..68b255ab 100644 --- a/papers/bmodes/scripts/plot_cosebis_filter_overlay.py +++ b/papers/bmodes/scripts/plot_cosebis_filter_overlay.py @@ -21,16 +21,16 @@ from astropy.io import fits from cosmo_numba.B_modes.cosebis import COSEBIS from plotting_utils import PAPER_MPLSTYLE +from pseudo_cl_io import load_pseudo_cl_data plt.style.use(PAPER_MPLSTYLE) def load_bb_data(pseudo_cl_path, pseudo_cov_path): """Load BB bandpower data and errorbars.""" - with fits.open(pseudo_cl_path) as hdul: - data = hdul["PSEUDO_CELL"].data - ell = np.asarray(data["ELL"], dtype=float) - bb = np.asarray(data["BB"], dtype=float) + data = load_pseudo_cl_data(pseudo_cl_path) + ell = np.asarray(data["ELL"], dtype=float) + bb = np.asarray(data["BB"], dtype=float) with fits.open(pseudo_cov_path) as hdul: cov_bb = hdul["COVAR_BB_BB"].data diff --git a/papers/bmodes/scripts/pseudo_cl_io.py b/papers/bmodes/scripts/pseudo_cl_io.py new file mode 100644 index 00000000..b7b1c796 --- /dev/null +++ b/papers/bmodes/scripts/pseudo_cl_io.py @@ -0,0 +1,19 @@ +"""Read pseudo-Cℓ parts for the B-modes paper workflow.""" + +from sp_validation import sacc_io + + +def load_pseudo_cl_data(path): + """Return a pseudo-Cℓ part's spectra as ``{"ELL", "EE", "EB", "BB"}`` arrays. + + The bandpower covariance is a separate FITS product (``pseudo_cl_cov``). + Loading goes through ``sacc_io.load``, so an unblinded real-data part is + refused. + """ + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(sacc_io.load(str(path)), (0, 0)) + missing = [name for name, values in (("BB", bb), ("EB", eb)) if values is None] + if missing: + raise ValueError( + f"{path} lacks required pseudo-Cℓ spectra: {', '.join(missing)}" + ) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index dd7219a3..211214e3 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -2,8 +2,8 @@ Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and runs the same ``run_2pcf.run_2pcf`` compute the fiducial two_point recipes call, -once per version, writing every version's ξ± text dump (+ ξ+/ξ- FITS) into one -lc ``{output}`` dir under run_2pcf's native, already-canonical name +once per version, writing every version's ξ± text dump into one lc ``{output}`` +dir under run_2pcf's native, already-canonical name ``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.txt`` — the exact pattern ``cosebis_version_comparison._xi_integration`` reconstructs. @@ -76,7 +76,7 @@ def _from_cli(argv=None): ver=ver, cat_config=a.cat_config, output_dir=a.out, - save_fits=True, + grid=grid, **GRIDS[grid], ) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index a0323b7c..ae99aba4 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -74,11 +74,11 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # Pure E/B-mode decomposition (config space) - pure_eb: - min_sep_int: 0.08 - max_sep_int: 300 - nbins_int: 1000 + # The fine ξ± grid the B-mode integrals run over. + integration: + min_sep: 0.08 + max_sep: 300 + nbins: 1000 # COSEBIs decomposition (config space, fine integration binning) cosebis: diff --git a/pyproject.toml b/pyproject.toml index 4ad6fbf0..632793f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,8 @@ dependencies = [ "pymaster", "regions", "reproject", - "sacc>=0.12", + # Floor matches the resolved lock; needs sacc's 2.x rewrite (BlockDiagonalCovariance). + "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to # shape-(1,) `array([x])`. camb's BBN Y_He predictor (bbn.py) wraps the diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 33113370..7da61279 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -15,6 +15,7 @@ find_conservative_scale_cut_key, ) from ..statistics import chi2_and_pte +from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin from .cosebis import CosebisMixin from .pseudo_cl import PseudoClMixin @@ -22,8 +23,42 @@ from .pure_eb import PureEBMixin from .real_space import RealSpaceMixin - # %% +BMODE_COLUMNS = { + "xip_B": r"xi+B", + "xim_B": r"xi-B", + "combined": "Combined", + "COSEBIS": "COSEBIS", + "C_l_BB": "C_l^BB", +} + + +def print_bmode_summary(summary, fiducial_scale_cut, cov_methods=()): + """Print the B-mode PTE table for ``{version: {statistic: pte}}``. + + Statistics absent from a row print as ``--``. + """ + sc_label = f"[{fiducial_scale_cut[0]}-{fiducial_scale_cut[1]} arcmin]" + sep = "\u2500" * 70 + header = f"{'Version':<28s}" + "".join( + f"{label:>10s}" for label in BMODE_COLUMNS.values() + ) + + print(f"\nB-mode summary {sc_label}") + print(sep) + print(header) + print(sep) + for ver, row in summary.items(): + cells = "".join( + f"{row[s]:>10.4f}" if s in row else f"{'--':>10s}" for s in BMODE_COLUMNS + ) + print(f"{ver:<28s}{cells}") + print(sep) + if cov_methods: + print(f"Covariance: {', '.join(sorted(cov_methods))}") + print() + + class CosmologyValidation( CosebisMixin, PureEBMixin, @@ -380,6 +415,21 @@ def _output_path(self, *parts): """ return os.path.abspath(os.path.join(self.cc["paths"]["output"], *parts)) + def sacc_nz(self, version): + """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. + + The round is single-bin, so the whole survey n(z) is bin 0. + """ + return {0: tuple(self.get_redshift(version))} + + def sacc_metadata(self, version): + """Provenance metadata stored on every SACC part for ``version``.""" + return { + "catalogue_version": version, + "sp_validation_version": __version__, + "npatch": self.npatch, + } + def get_redshift(self, version): """Load redshift distribution for a catalog version. @@ -602,37 +652,5 @@ def summarize_bmodes(self, fiducial_scale_cut=(12, 83), versions=None): summary[ver] = row - # Print summary table - col_labels = { - "xip_B": r"xi+B", - "xim_B": r"xi-B", - "combined": "Combined", - "COSEBIS": "COSEBIS", - "C_l_BB": "C_l^BB", - } - stats_order = list(col_labels) - - sc_label = f"[{fiducial_scale_cut[0]}-{fiducial_scale_cut[1]} arcmin]" - sep = "\u2500" * 70 - header = f"{'Version':<28s}" + "".join( - f"{label:>10s}" for label in col_labels.values() - ) - - print(f"\nB-mode summary {sc_label}") - print(sep) - print(header) - print(sep) - - for ver in versions: - row = summary[ver] - cells = "".join( - f"{row[s]:>10.4f}" if s in row else f"{'--':>10s}" for s in stats_order - ) - print(f"{ver:<28s}{cells}") - - print(sep) - if cov_methods: - print(f"Covariance: {', '.join(sorted(cov_methods))}") - print() - + print_bmode_summary(summary, fiducial_scale_cut, cov_methods) return summary diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 514049a6..cf74efc6 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -17,6 +17,7 @@ from astropy.io import fits from cs_util.cosmo import get_theo_c_ell +from .. import sacc_io from ..pseudo_cl import ( apply_random_rotation, get_n_gal_map, @@ -26,6 +27,47 @@ ) from ..rho_tau import get_params_rho_tau from ..statistics import chi2_and_pte, cov_from_one_covariance +from .sacc_writers import BIN as SACC_BIN +from .sacc_writers import pseudo_cl_to_sacc + + +def plot_pseudo_cl_spectrum(datasets, spectrum, output_path): + """Two-panel ℓC_ℓ / C_ℓ figure for one spectrum across catalogue versions. + + ``datasets`` maps a version to ``{"ell", "cl", "cov", "style"}``, where + ``style`` carries the ``marker`` and ``colour`` the version is drawn with. + """ + fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) + minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] + + for panel, scaled in ((ax[0], True), (ax[1], False)): + for version, data in datasets.items(): + ell, cl = np.asarray(data["ell"]), np.asarray(data["cl"]) + err = np.sqrt(np.diag(np.asarray(data["cov"]))) + style = data.get("style", {}) + panel.errorbar( + ell, + ell * cl if scaled else cl, + yerr=ell * err if scaled else err, + fmt=style.get("marker", "."), + color=style.get("colour"), + label=f"{version} {spectrum}", + capsize=2 if scaled else None, + ) + panel.set_ylabel(r"$\ell C_\ell$" if scaled else r"$C_\ell$") + panel.set_xlim(ell.min() - 10, ell.max() + 100) + panel.set_xscale("squareroot") + panel.set_xticks(np.array([100, 400, 900, 1600])) + panel.minorticks_on() + panel.tick_params(axis="x", which="minor", length=2, width=0.8) + panel.xaxis.set_ticks(minor_ticks, minor=True) + + ax[1].set_xlabel(r"$\ell$") + ax[1].set_yscale("log") + plt.suptitle(f"Pseudo-Cl {spectrum} (Gaussian covariance)") + plt.legend() + plt.savefig(output_path) + plt.close(fig) class PseudoClMixin: @@ -451,14 +493,23 @@ def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" ) - def calculate_pseudo_cl(self): + def calculate_pseudo_cl(self, out_path=None): """ Compute the pseudo-Cl of given catalogs. + + ``out_path`` is the exact destination the part is born at (one version + only); ``None`` defaults each part to ``pseudo_cl_{ver}.sacc``. """ self.print_start("Computing pseudo-Cl's") nside = self.nside + if out_path is not None and len(self.versions) != 1: + raise ValueError( + "calculate_pseudo_cl(out_path=...) writes one part to one path, " + f"but {len(self.versions)} versions are configured; call per version" + ) + try: self._pseudo_cls except AttributeError: @@ -468,20 +519,32 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_{ver}.fits") - if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + ver_out_path = out_path or self._output_path(f"pseudo_cl_{ver}.sacc") + if os.path.exists(ver_out_path): + self.print_done( + f"Skipping Pseudo-Cl's calculation, {ver_out_path} exists" + ) + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc( + ver_out_path + ) elif self.cell_method == "map": - self.calculate_pseudo_cl_map(ver, nside, out_path) + self.calculate_pseudo_cl_map(ver, nside, ver_out_path) elif self.cell_method == "catalog": - self.calculate_pseudo_cl_catalog(ver, out_path) + self.calculate_pseudo_cl_catalog(ver, ver_out_path) else: raise ValueError(f"Unknown cell method: {self.cell_method}") self.print_done("Done pseudo-Cl's") + @staticmethod + def _load_pseudo_cl_sacc(out_path): + """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict.""" + # Readback of a part this producer just wrote — a legitimate pre-blind + # consumer, so the fail-closed load is opted out of. + s = sacc_io.load(out_path, allow_unblinded=True) + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} + def calculate_pseudo_cl_map(self, ver, nside, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -547,10 +610,9 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def calculate_pseudo_cl_catalog(self, ver, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -563,10 +625,9 @@ def calculate_pseudo_cl_catalog(self, ver, out_path): ) self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def get_n_gal_map(self, params, nside, cat_gal): """Weighted galaxy number-density map (thin wrapper -> primitive).""" @@ -655,223 +716,50 @@ def apply_random_rotation(self, e1, e2, rng=None): """ return apply_random_rotation(e1, e2, rng) - def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): - """ - Save pseudo-Cl's to a FITS file. + def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): + """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). - Parameters - ---------- - pseudo_cl : np.array - Pseudo-Cl's to save. - out_path : str - Path to save the pseudo-Cl's to. + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); + the writer takes the shared bandpower window from ``wsp``. No covariance + is attached here. """ - # Create columns of the fits file - col1 = fits.Column(name="ELL", format="D", array=ell_eff) - col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) - col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) - col4 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) - coldefs = fits.ColDefs([col1, col2, col3, col4]) - cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") - - cell_hdu.writeto(out_path, overwrite=True) + s = pseudo_cl_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + ell_eff, + cl_all, + wsp, + ) + sacc_io.save(s, out_path, type="data") def plot_pseudo_cl(self): - """ - Plot pseudo-Cl's for given catalogs. - """ + """Plot the EE/EB/BB pseudo-Cl spectra for every version.""" self.print_cyan("Plotting pseudo-Cl's") - # Plotting EE - out_path = self._output_path("cell_ee.png") - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data - ax[0].errorbar( - ell, - ell * self.pseudo_cls[ver]["pseudo_cl"]["EE"], - yerr=ell * np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EE", - color=self.cc[ver]["colour"], - capsize=2, - ) - - ax[0].set_ylabel(r"$\ell C_\ell$") - - ax[0].set_xlim(ell.min() - 10, ell.max() + 100) - ax[0].set_xscale("squareroot") - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data - ax[1].errorbar( - ell, - self.pseudo_cls[ver]["pseudo_cl"]["EE"], - yerr=np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EE", - color=self.cc[ver]["colour"], - ) - - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") - - ax[1].set_xlim(ell.min() - 10, ell.max() + 100) - ax[1].set_xscale("squareroot") - ax[1].set_yscale("log") - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle("Pseudo-Cl EE (Gaussian covariance)") - plt.legend() - plt.savefig(out_path) - - # Plotting EB - out_path = self._output_path("cell_eb.png") - - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EB_EB"].data - ax[0].errorbar( - ell, - ell * self.pseudo_cls[ver]["pseudo_cl"]["EB"], - yerr=ell * np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EB", - color=self.cc[ver]["colour"], - capsize=2, + for spectrum in ("EE", "EB", "BB"): + datasets = { + ver: { + "ell": self.pseudo_cls[ver]["pseudo_cl"]["ELL"], + "cl": self.pseudo_cls[ver]["pseudo_cl"][spectrum], + "cov": self.pseudo_cls[ver]["cov"][ + f"COVAR_{spectrum}_{spectrum}" + ].data, + "style": { + "marker": self.cc[ver]["marker"], + "colour": self.cc[ver]["colour"], + }, + } + for ver in self.versions + } + plot_pseudo_cl_spectrum( + datasets, spectrum, self._output_path(f"cell_{spectrum.lower()}.png") ) - ax[0].axhline(0, color="black", linestyle="--") - ax[0].set_ylabel(r"$\ell C_\ell$") - - ax[0].set_xlim(ell.min() - 10, ell.max() + 100) - ax[0].set_xscale("squareroot") - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_EB_EB"].data - ax[1].errorbar( - ell, - self.pseudo_cls[ver]["pseudo_cl"]["EB"], - yerr=np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " EB", - color=self.cc[ver]["colour"], - ) - - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") - - ax[1].set_xlim(ell.min() - 10, ell.max() + 100) - ax[1].set_xscale("squareroot") - ax[1].set_yscale("log") - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle("Pseudo-Cl EB (Gaussian covariance)") - plt.legend() - plt.savefig(out_path) - - # Plotting BB - out_path = self._output_path("cell_bb.png") - - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data - ax[0].errorbar( - ell, - ell * self.pseudo_cls[ver]["pseudo_cl"]["BB"], - yerr=ell * np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " BB", - color=self.cc[ver]["colour"], - capsize=2, - ) - - ax[0].axhline(0, color="black", linestyle="--") - ax[0].set_ylabel(r"$\ell C_\ell$") - - ax[0].set_xlim(ell.min() - 10, ell.max() + 100) - ax[0].set_xscale("squareroot") - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - cov = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data - ax[1].errorbar( - ell, - self.pseudo_cls[ver]["pseudo_cl"]["BB"], - yerr=np.sqrt(np.diag(cov)), - fmt=self.cc[ver]["marker"], - label=ver + " BB", - color=self.cc[ver]["colour"], - ) - - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") - - ax[1].set_xlim(ell.min() - 10, ell.max() + 100) - ax[1].set_xscale("squareroot") - ax[1].set_yscale("log") - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle("Pseudo-Cl BB (Gaussian covariance)") - plt.legend() - plt.savefig(out_path) - - # Print C_l^BB PTE for each version and save BB data - print("\nC_l^BB PTE summary:") for ver in self.versions: cl_bb = self.pseudo_cls[ver]["pseudo_cl"]["BB"] cov_bb = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data chi2_bb, _, pte_bb = chi2_and_pte(cl_bb, cov_bb) - chi2_bb = float(chi2_bb) print( f" {ver}: C_l^BB PTE = {pte_bb:.4f} " - f"(chi2/dof = {chi2_bb:.1f}/{len(cl_bb)})" - ) - - # Save BB data + covariance to .npz - ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] - bb_out = self._output_path(f"{ver}_cell_bb_data.npz") - np.savez( - bb_out, - ell=ell, - cl_bb=cl_bb, - cov_bb=cov_bb, - chi2_bb=np.array(chi2_bb), - pte_bb=np.array(pte_bb), + f"(chi2/dof = {float(chi2_bb):.1f}/{len(cl_bb)})" ) - print(f" Saved BB data to {bb_out}") diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 98527490..f8002244 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -16,10 +16,12 @@ from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat +from .. import sacc_io from ..rho_tau import ( get_rho_tau_w_cov, get_samples, ) +from .sacc_writers import rho_tau_to_sacc class PSFSystematicsMixin: @@ -41,11 +43,42 @@ def calculate_rho_tau_stats(self): cov_rho=self.compute_cov_rho, npatch=self.npatch, ) + self.rho_tau_to_sacc_part( + ver, out_dir, base, rho_stat_handler, tau_stat_handler + ) self.print_done("Rho stats finished") self._rho_stat_handler = rho_stat_handler self._tau_stat_handler = tau_stat_handler + def rho_tau_to_sacc_part( + self, version, out_dir, base, rho_stat_handler, tau_stat_handler + ): + """Write the ρ/τ SACC part for one version. + + ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The + ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` is passed as + ``tau_cov_th`` when it exists; without it the τ block falls back — + loudly — to a variance diagonal. + """ + tau_cov_path = os.path.join(out_dir, f"cov_tau_{base}_th.npy") + tau_cov_th = np.load(tau_cov_path) if os.path.exists(tau_cov_path) else None + if tau_cov_th is None: + self.print_magenta( + f"No τ theory covariance at {tau_cov_path}; writing ρ/τ SACC part " + "with a diagonal placeholder covariance (τ inference block is a " + "variance diagonal, not CovTauTh)." + ) + s = rho_tau_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + rho_stat_handler.rho_stats, + tau_stat_handler.tau_stats, + tau_cov_th=tau_cov_th, + ) + out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") + sacc_io.save(s, out_path, type="data") + @property def rho_stat_handler(self): if not hasattr(self, "_rho_stat_handler"): diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 76d05d0a..1521ecc7 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -12,12 +12,11 @@ import matplotlib.ticker as mticker import numpy as np import treecorr -from astropy.io import fits from cs_util import plots as cs_plots class RealSpaceMixin: - def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): + def calculate_2pcf(self, ver, npatch=None, **treecorr_config): """ Calculate the two-point correlation function (2PCF) ξ± for a given catalog version with TreeCorr. @@ -34,9 +33,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): npatch (int, optional): The number of patches to use for the calculation. Defaults to the instance's `npatch` attribute. - save_fits (bool, optional): Whether to save the ξ± results to FITS files. - Defaults to False. - **treecorr_config: Additional TreeCorr configuration parameters that will override the instance's default `treecorr_config`. For example, `min_sep=1`. @@ -49,8 +45,7 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): calculation is skipped, and the results are loaded from the file. - If a patch file for the given configuration does not exist, it is created during the process. - - FITS files for ξ+ and ξ− are saved with additional metadata in their - headers if `save_fits` is True. + - The ``.txt`` TreeCorr dump is the only raw byproduct written here. """ self.print_magenta(f"Computing {ver} ξ±") @@ -99,75 +94,13 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): # Process the catalog & write the correlation functions gg.process(cat_gal) - gg.write(out_fname, write_patch_results=True, write_cov=True) - - # Save xi_p and xi_m results to fits file - # (moved outside so it runs even if txt exists) - if save_fits: - lst = np.arange(1, treecorr_config["nbins"] + 1) - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.rnom) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - # Use same naming format as txt output - fits_base = out_fname.replace(".txt", "").replace("_xi_", "_") - xiplus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_plus_{ver}')}.fits", - overwrite=True, - ) - - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_minus_{ver}')}.fits", - overwrite=True, - ) + # Never write_patch_results: a per-patch ξ± realisation is an + # unblinded data vector, and nothing downstream reads one — the + # covariance a consumer needs is the matrix, which the SACC part + # carries. The .txt keeps the matrix only where there are patches to + # estimate it from; at npatch=1 var_method is "shot" and it would add + # nothing over the varxip/varxim columns. + gg.write(out_fname, write_patch_results=False, write_cov=int(npatch) > 1) # Add correlation object to class if not hasattr(self, "cat_ggs"): diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py new file mode 100644 index 00000000..8b7f57b7 --- /dev/null +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -0,0 +1,237 @@ +"""Born-as-SACC writers for the cosmo_val data products. + +A thin, pure layer between the ``cosmo_val`` mixins (which compute statistics as +TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which +knows the file layout). Each ``*_to_sacc`` function turns one already-computed +statistic into a single-statistic SACC — a *part* — carrying that statistic's +own covariance as its one block. :func:`assemble_analysis_sacc` rebuilds the +single ``{version}.sacc`` analysis file from these parts. + +Everything here is single-bin today (``bins=(0, 0)``); the interface is +tomography-native so a future round supplies real bin pairs unchanged. +""" + +import numpy as np +import sacc + +from .. import sacc_io as sio +from ..pseudo_cl import bandpower_window_from_workspace + +# Statistics carried in the analysis file, and their custom-type k indices. +RHO_K = range(6) # ρ_0 … ρ_5 +TAU_K = (0, 2, 5) # τ_0, τ_2, τ_5 + +# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). +_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 + +BIN = (0, 0) + + +def xi_to_sacc( + nz, + metadata, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, + variances=None, + covariance=None, +): + """One ξ± part (``bins=(0, 0)``) on a named angular grid. + + The grid's covariance comes in one of two shapes: ``covariance``, the dense + ``[ξ+; ξ−]``-ordered block (a jackknife estimate), or ``variances``, the + concatenated ``[varxip; varxim]`` diagonal. At most one may be given. + """ + s = sio.new_sacc(nz, metadata) + sio.add_xi( + s, + BIN, + theta, + xip, + xim, + grid=grid, + theta_nom=theta_nom, + npairs=npairs, + weight=weight, + ) + if covariance is not None and variances is not None: + raise ValueError("give xi_to_sacc a dense covariance or variances, not both") + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + elif variances is not None: + sio.add_diagonal_covariance(s, np.asarray(variances)) + return s + + +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): + """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. + + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the + window comes from :func:`bandpower_window_from_workspace`. ``covariance``, + when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. + """ + window_ells, window_weights = bandpower_window_from_workspace(wsp) + s = sio.new_sacc(nz, metadata) + sio.add_pseudo_cl( + s, + BIN, + ell_eff, + cl_all[_NMT_EE], + cl_all[_NMT_BB], + cl_all[_NMT_EB], + window_ells=window_ells, + window_weights=window_weights, + ) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def cosebis_to_sacc(nz, metadata, result, scale_cut): + """One COSEBIs part at the fiducial scale cut. + + ``result`` is a single scale-cut result dict from + ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` + is the ``[En; Bn]``-ordered COSEBIs covariance. + """ + s = sio.new_sacc(nz, metadata) + sio.add_cosebis(s, BIN, result["En"], scale_cut, Bn=result["Bn"]) + s.add_covariance(np.asarray(result["cov"])) + return s + + +def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): + """One pure-E/B part: the six ``sacc_io.PURE_KEYS`` blocks. + + ``eb`` is a mapping with the six keys (``xip_E`` … ``xim_amb``); each array + is sampled at ``theta``. ``covariance``, when given, is the dense block in + ``PURE_KEYS`` order (matching ``b_modes._EB_KEYS`` and the insertion order). + """ + s = sio.new_sacc(nz, metadata) + sio.add_pure_eb(s, BIN, theta, **{key: eb[key] for key in sio.PURE_KEYS}) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): + """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. + + ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables + (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, … and the τ analogue). + ρ carries a ``varrho`` diagonal; τ carries a ``vartau`` diagonal, with + ``tau_cov_th`` — a ``(3·nbin, 3·nbin)`` k-major matrix over the τ-plus points + only — scattered into the τ-plus rows/columns when given. ``tau_cov_th=None`` + leaves the τ block fully diagonal. + """ + s = sio.new_sacc(nz, metadata) + theta_rho = np.asarray(rho_stats["theta"]) + for k in RHO_K: + sio.add_rho( + s, + k, + theta_rho, + np.asarray(rho_stats[f"rho_{k}_p"]), + np.asarray(rho_stats[f"rho_{k}_m"]), + ) + theta_tau = np.asarray(tau_stats["theta"]) + for k in TAU_K: + sio.add_tau( + s, + BIN, + k, + theta_tau, + np.asarray(tau_stats[f"tau_{k}_p"]), + np.asarray(tau_stats[f"tau_{k}_m"]), + ) + nbin = len(theta_tau) + rho_var = np.concatenate( + [ + np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) + for k in RHO_K + ] + ) + tau_var = np.concatenate( + [ + np.concatenate([tau_stats[f"vartau_{k}_p"], tau_stats[f"vartau_{k}_m"]]) + for k in TAU_K + ] + ) + if tau_cov_th is None: + s.add_covariance(np.concatenate([rho_var, tau_var])) + return s + tau_cov_th = np.asarray(tau_cov_th) + n_plus = len(TAU_K) * nbin + if tau_cov_th.shape != (n_plus, n_plus): + raise ValueError( + f"tau_cov_th shape {tau_cov_th.shape} does not match the " + f"{n_plus} τ-plus points ({len(TAU_K)} indices × {nbin} bins) — " + "CovTauTh.build_cov returns one (plus-folded) component per τ index" + ) + n_rho, n_tau = len(rho_var), len(tau_var) + tau_block = np.diag(tau_var) + # τ-plus local positions in the τ block, k-major (per-k layout is [+; −]). + plus = np.concatenate( + [np.arange(2 * i * nbin, 2 * i * nbin + nbin) for i in range(len(TAU_K))] + ) + tau_block[np.ix_(plus, plus)] = tau_cov_th + full = np.zeros((n_rho + n_tau, n_rho + n_tau)) + full[:n_rho, :n_rho] = np.diag(rho_var) + full[n_rho:, n_rho:] = tau_block + s.add_covariance(full) + return s + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _copy_data_points(dst, src): + """Append every data point of ``src`` into ``dst`` (tags preserved).""" + for dp in src.data: + dst.add_data_point(dp.data_type, dp.tracers, dp.value, **dp.tags) + + +def assemble_analysis_sacc(parts): + """Rebuild the single ``{version}.sacc`` analysis file from parts. + + Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded + from disk) carrying its own covariance = its block. Tracers and metadata are + seeded from ``parts[0]`` (every part describes the same catalogue version). + Data points are re-added in the order the parts are given, which must be the + canonical order (ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ), and the + per-part blocks become one ``BlockDiagonalCovariance``. Insertion order and + block order therefore agree by construction — validated by + :func:`sp_validation.sacc_io.assemble_covariance`. + + Parameters + ---------- + parts : sequence of sacc.Sacc + Single-statistic parts, each with a covariance, in canonical order. + + Returns + ------- + sacc.Sacc + The analysis Sacc with a ``BlockDiagonalCovariance`` covering every point. + """ + s = sacc.Sacc() + s.tracers.update(parts[0].tracers) + s.metadata.update(parts[0].metadata) + blocks = [] + cursor = 0 + for part in parts: + if part.covariance is None: + raise ValueError( + "every analysis part must carry its own covariance block; " + f"a part with data types {sorted(set(dp.data_type for dp in part.data))} " + "has none" + ) + n = len(part.mean) + _copy_data_points(s, part) + blocks.append((np.arange(cursor, cursor + n), part.covariance.dense)) + cursor += n + return sio.assemble_covariance(s, blocks) diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index c9355ec9..34cbc68d 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -280,3 +280,32 @@ def get_pseudo_cls_catalog( cl_all = wsp.decouple_cell(cl_coupled) return ell_eff, cl_all, wsp + + +# NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. +_NMT_EE = 0 + + +def bandpower_window_from_workspace(wsp): + """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. + + NaMaster's ``get_bandpower_windows()`` returns a four-index array + ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower + is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's + ``BandpowerWindow`` model (one window per bandpower, shared across the + stored spectra) needs the per-spectrum *decoupling* window, i.e. the + diagonal EE←EE block (equal to BB←BB and EB←EB, verified identical). + + Returns + ------- + window_ells : np.ndarray + Multipoles the window spans, ``arange(n_ell)`` — the ``ell`` axis of + ``compute_coupled_cell``. + window_weights : np.ndarray + ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + """ + bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) + diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) + window_ells = np.arange(diagonal.shape[1], dtype=float) + return window_ells, diagonal.T diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py new file mode 100644 index 00000000..f17d5468 --- /dev/null +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -0,0 +1,300 @@ +"""Integration tests for the ``assemble_sacc.py`` workflow script. + +The pure assembler (``sacc_writers.assemble_analysis_sacc``) is covered in +``test_sacc_writers.py``. This file exercises the *script seam* the DAG uses: +``assemble_sacc.assemble_sacc`` loads per-statistic ``.sacc`` part *files* in +CANONICAL order, injects the born-cov-less ξ± / pseudo-Cℓ blocks from the real +CosmoCov ``.txt`` and NaMaster covariance FITS, and writes one +``{version}.sacc`` whose points and covariance blocks land in canonical order. + +The script lives under ``workflow/scripts`` (off the package path); it is loaded +by file path exactly as the lightcone/ASTRA CLI path imports it. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _load_assemble_module(): + """Import ``workflow/scripts/assemble_sacc.py`` by file path.""" + repo_root = next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + path = repo_root / "workflow" / "scripts" / "assemble_sacc.py" + spec = importlib.util.spec_from_file_location("assemble_sacc", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +asm = _load_assemble_module() + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +def _xi_cov_txt(tmp_path, n=12, seed=21): + """A CosmoCov-format ξ± covariance: the dense ``_processed.txt`` matrix. + + Same writer/format ``covariance_process`` emits and ``--xi-cov`` reads, at + the synthetic parts' 12-point ([ξ+; ξ−] over 6 θ) size. Returns + ``(path, matrix)``. + """ + cov = _spd(n, seed) + path = tmp_path / "xi_cov_processed.txt" + np.savetxt(str(path), cov) + return str(path), cov + + +def _pseudo_cl_cov_fits(tmp_path, n=3): + """A NaMaster covariance FITS: one HDU per spectrum. Returns (path, blocks).""" + from astropy.io import fits + + blocks = {"EE": _spd(n, 31), "BB": _spd(n, 32), "EB": _spd(n, 33)} + path = tmp_path / "pseudo_cl_cov.fits" + fits.HDUList( + [fits.PrimaryHDU()] + + [fits.ImageHDU(block, name=f"COVAR_{k}_{k}") for k, block in blocks.items()] + ).writeto(str(path)) + return str(path), blocks + + +def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_reporting",)): + """Write per-statistic parts to disk; return the ``{name: path}`` mapping. + + Parts named in ``cov_less`` are written without a covariance (mimicking the + born-cov-less ξ± reporting / pseudo-Cℓ parts); the rest carry their own block. + """ + nz = {0: _nz()} + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + if "xi_reporting" not in cov_less: + xi.add_covariance(_spd(len(xi.mean), 1)) + + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc( + nz, + META, + ell, + cl_all, + _Wsp(), + covariance=None if "pseudo_cl" in cov_less else _spd(9, 2), + ) + + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + + eb_arrays = { + key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS) + } + eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + + rho = {"theta": theta} + tau = {"theta": theta} + rng = np.random.default_rng(5) + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + + parts = { + "xi_reporting": xi, + "pseudo_cl": cl, + "cosebis": co, + "pure_eb": eb, + "rho_tau": rt, + } + if not with_pseudo_cl: + parts.pop("pseudo_cl") + + paths = {} + for name, part in parts.items(): + p = tmp_path / f"{name}.sacc" + sio.save(part, str(p), type="mock") + paths[name] = str(p) + return paths + + +def test_assemble_sacc_canonical_order(tmp_path): + """Every point is covered and the blocks land in canonical order + (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) + cov_path, xi_cov = _xi_cov_txt(tmp_path) + cl_cov_path, _blocks = _pseudo_cl_cov_fits(tmp_path) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cl_cov_path + ) + assert out.exists() + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + # Canonical insertion order: the first data types are ξ+ then ξ−. + types_in_order = [dp.data_type for dp in s.data] + assert types_in_order[0] == sio.XI_PLUS + assert sio.XI_MINUS in types_in_order + # ξ appears before pseudo-Cℓ before COSEBIs before pure-E/B before ρ/τ. + first = {t: types_in_order.index(t) for t in set(types_in_order)} + assert first[sio.XI_PLUS] < first[sio.CL_EE] < first[sio.COSEBI_EE] + assert first[sio.COSEBI_EE] < first[sio.PURE_TYPES["xip_E"]] + assert first[sio.PURE_TYPES["xip_E"]] < first[sio.RHO_PLUS.format(k=0)] + assert first[sio.RHO_PLUS.format(k=0)] < first[sio.TAU_PLUS.format(k=0)] + + # The ξ± block is the injected CosmoCov matrix on its own points. + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + dense = s.covariance.dense + assert np.allclose(dense[np.ix_(xi_idx, xi_idx)], xi_cov) + # ...and it does not bleed into the neighbouring COSEBIs block (cross zero). + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert np.allclose(dense[np.ix_(xi_idx, co_idx)], 0.0) + + +def test_injected_xi_covariance_replaces_the_parts_own(tmp_path): + """The analytic ξ± covariance wins over the estimate the part was born with. + + The reporting part carries the jackknife it was measured with — useful as a + diagnostic, but the analysis file takes the CosmoCov block. + """ + paths = _write_parts(tmp_path, cov_less=()) # ξ± born with its own jackknife + cov_path, xi_cov = _xi_cov_txt(tmp_path) + cl_cov_path, _blocks = _pseudo_cl_cov_fits(tmp_path) + + born = sio.load(paths["xi_reporting"], allow_unblinded=True).covariance.dense + assert not np.allclose(born, xi_cov) # the two are distinguishable + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cl_cov_path + ) + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) + + +def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): + """The NaMaster cov FITS (COVAR_EE_EE/BB_BB/EB_EB) → block-diagonal pseudo-Cℓ + block, beside the injected CosmoCov ξ± block (the live default).""" + paths = _write_parts(tmp_path, cov_less=("xi_reporting", "pseudo_cl")) + cov_path, xi_cov = _xi_cov_txt(tmp_path) + # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. + cov_fits, blocks = _pseudo_cl_cov_fits(tmp_path) + ee, bb, eb = blocks["EE"], blocks["BB"], blocks["EB"] + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), xi_cov=cov_path, pseudo_cl_cov=cov_fits + ) + tr = ("source_0", "source_0") + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + dense = s.covariance.dense + expected = np.zeros((9, 9)) + expected[0:3, 0:3], expected[3:6, 3:6], expected[6:9, 6:9] = ee, bb, eb + assert np.allclose(dense[np.ix_(cl_idx, cl_idx)], expected) + # ξ± carries its own CosmoCov block; the two don't bleed into each other. + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(dense[np.ix_(xi_idx, xi_idx)], xi_cov) + assert np.allclose(dense[np.ix_(xi_idx, cl_idx)], 0.0) + + +def test_missing_injected_covariance_raises(tmp_path): + """A statistic whose covariance is external cannot fall back to its own.""" + paths = _write_parts(tmp_path, cov_less=()) # every part born with a block + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="takes its analysis covariance from"): + asm.assemble_sacc("vSYNTH", paths, str(out)) + + +def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): + """With pseudo_cl absent, assembly still succeeds and omits the Cℓ points.""" + paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_reporting",)) + assert "pseudo_cl" not in paths + cov_path, _xi_cov = _xi_cov_txt(tmp_path) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=cov_path) + tr = ("source_0", "source_0") + assert len(s.indices(sio.CL_EE, tr)) == 0 + # Round-trips as a valid BlockDiagonalCovariance over the remaining points. + s2 = sio.load(str(out)) + assert type(s2.covariance).__name__ == "BlockDiagonalCovariance" + assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) + + +def test_assemble_sacc_expected_part_missing_raises(tmp_path): + """A typo'd input keyword drops a part from part_paths; the expected list + catches it rather than silently omitting the statistic.""" + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) + # Simulate a rule-input typo: cosebis wired under the wrong key. + paths["cosebi"] = paths.pop("cosebis") + cov_path, _xi_cov = _xi_cov_txt(tmp_path) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="expected parts \\['cosebis'\\] missing"): + asm.assemble_sacc( + "vSYNTH", + paths, + str(out), + expected=["xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], + xi_cov=cov_path, + ) + + +def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): + """A typo in the expected list itself is rejected (not a valid statistic).""" + paths = _write_parts(tmp_path, cov_less=("xi_reporting",)) + cov_path, _xi_cov = _xi_cov_txt(tmp_path) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="not assemblable statistics"): + asm.assemble_sacc( + "vSYNTH", paths, str(out), expected=["cosebi"], xi_cov=cov_path + ) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 68c09981..3b7eecae 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -1,11 +1,15 @@ -"""Back-pressure guard #2: the B-modes Snakemake workflow dry-runs. +"""Back-pressure guard #2: the paper Snakemake workflows dry-run. -The reorg is allowed to change the rule graph; this guard only asserts that -Snakemake can still parse the workflow and construct a dry run. +The reorg is allowed to change the rule graph; these guards only assert that +Snakemake can still parse each composed workflow and construct a dry run. One +guard covers papers/bmodes (config space, no cosmo_val block); a second covers +papers/cosmo_val, whose config DOES carry a cosmo_val block — so it is the only +one that includes cosmo_val.smk and hence the born-as-SACC + assemble rules. """ import os import subprocess +import sys from pathlib import Path import pytest @@ -26,31 +30,59 @@ def _repo_root() -> Path: raise RuntimeError("could not locate repo root (no pyproject.toml above test)") -@requires_candide_data -def test_bmodes_workflow_dry_runs(): - """The paper B-mode workflow must still parse and dry-run cleanly.""" - workflow_dir = _repo_root() / "papers/bmodes" - # PYTHONUNBUFFERED satisfies the Snakefile's `envvars:` declaration without - # depending on the invoking shell's environment. +def _dry_run(workflow_dir, targets, *extra_snakemake_args): + """Construct a dry run of the paper workflow at ``workflow_dir``. + + PYTHONUNBUFFERED satisfies the Snakefile's ``envvars:`` declaration. A dry + run never dispatches jobs, so any inherited SNAKEMAKE_PROFILE is dropped + rather than requiring its executor plugin. snakemake is invoked through + sys.executable, since a bare python3.12 may resolve off PATH to an + interpreter without it. + """ env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} - result = subprocess.run( + env.pop("SNAKEMAKE_PROFILE", None) + return subprocess.run( [ - "python3.12", + sys.executable, "-m", "snakemake", - "all_tapestry", + *targets, "--dry-run", "--cores", "1", "--configfile", "config/config.yaml", + *extra_snakemake_args, ], cwd=workflow_dir, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=60, + timeout=120, check=False, ) + + +@requires_candide_data +def test_bmodes_workflow_dry_runs(): + """The paper B-mode workflow must still parse and dry-run cleanly.""" + result = _dry_run(_repo_root() / "papers/bmodes", ["all_tapestry"]) + assert result.returncode == 0, result.stdout + + +@requires_candide_data +def test_cosmo_val_workflow_assemble_dry_runs(): + """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the + born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov.""" + version = "SP_v1.4.6.3_leak_corr" + result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) assert result.returncode == 0, result.stdout + # assemble_sacc must pull the tagged pseudo-Cl part + its NaMaster + # covariance (not the untagged cv_pseudo_cl diagnostic), plus every part. + out = result.stdout + assert "rule assemble_sacc:" in out, out + assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out + assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out + for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): + assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py new file mode 100644 index 00000000..ce601d25 --- /dev/null +++ b/src/sp_validation/tests/test_cli_seams.py @@ -0,0 +1,57 @@ +"""Smoke tests for workflow CLI seams — cheap guards against signature rot. + +The compute these scripts drive is cluster-only, so a removed or renamed kwarg +would only TypeError at invocation. Each test binds the exact call one seam +makes against the current signature (``inspect.signature(...).bind(...)``) — no +compute, no data — so the drift fails here instead of on the cluster. +""" + +import importlib.util +import inspect +from pathlib import Path + +import pytest + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def _load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_run_xi_sweep_run_2pcf_call_binds(): + """The kwargs run_xi_sweep passes to run_2pcf must bind to its signature.""" + root = _repo_root() + run_2pcf_mod = _load(root / "workflow/scripts/run_2pcf.py", "run_2pcf_seam") + sig = inspect.signature(run_2pcf_mod.run_2pcf) + # Exactly the keyword set run_xi_sweep._from_cli passes. + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + grid="integration", + min_sep=0.5, + max_sep=300.0, + nbins=1000, + npatch=1, + ) + # And the removed kwarg must NOT bind (guards against a silent re-add). + with pytest.raises(TypeError): + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + save_fits=True, + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 2b5ccf51..fd39c8e2 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -57,7 +57,9 @@ import pytest import yaml +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import BIN as SACC_BIN from sp_validation.rho_tau import get_params_rho_tau # These tests need the full harmonic-space stack (pymaster/NaMaster + healpy), @@ -112,6 +114,7 @@ def _write_synthetic_config(tmp_path): shear_cfg = { "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", "e2_col": "e2", @@ -509,24 +512,20 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): # calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): - """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. + """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise. save_pseudo_cl stores ELL/EE/EB/BB - (it drops the BE row); we pin the round-tripped table. + the same ~2e-12 catalog-path float noise; we pin the round-tripped spectra. """ ver = cv._test_version cv._pseudo_cls = {ver: {}} - out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") + out_path = cv._output_path(f"pseudo_cl_{ver}.sacc") cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - d = fits.getdata(out_path) - # FITS gives big-endian f8; normalize for value comparison. - ell = np.asarray(d["ELL"], dtype=np.float64) - ee = np.asarray(d["EE"], dtype=np.float64) - eb = np.asarray(d["EB"], dtype=np.float64) - bb = np.asarray(d["BB"], dtype=np.float64) + s = sacc_io.load(out_path, allow_unblinded=True) + ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + assert window is not None # the shared BandpowerWindow rides the part npt.assert_allclose( ell, @@ -591,3 +590,25 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): params = get_params_rho_tau(cv.cc[ver], survey=ver) _, cl_prim, _ = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) + + +def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): + """calculate_pseudo_cl(out_path=...) writes to the given path, never the + untagged native name — so the tagged and diagnostic rules stay disjoint.""" + ver = cv._test_version + cv._pseudo_cls = {} + tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") + native = cv._output_path(f"pseudo_cl_{ver}.sacc") + + cv.calculate_pseudo_cl(out_path=tagged) + + assert os.path.exists(tagged) + assert not os.path.exists(native) # no undeclared native basename touched + + +def test_calculate_pseudo_cl_out_path_rejects_multiversion(cv): + """out_path targets one part; a multi-version instance must fail loudly + rather than write every version to the same path.""" + cv.versions = [cv._test_version, "SecondVersion"] + with pytest.raises(ValueError, match="one part to one path"): + cv.calculate_pseudo_cl(out_path=cv._output_path("pseudo_cl_x.sacc")) diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py new file mode 100644 index 00000000..c4963b33 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -0,0 +1,389 @@ +"""Tests for :mod:`sp_validation.cosmo_val.sacc_writers`. + +Synthetic and fast: each ``*_to_sacc`` writer is exercised with in-memory +arrays, round-tripped through ``tmp_path``, and checked against the SACC layout +contract (data types, tags, ordering, covariance alignment). The analysis-file +assembler is verified to produce a single ``BlockDiagonalCovariance`` covering every +point with each per-statistic block correctly placed. One real small-nside +NaMaster round-trip proves the pseudo-Cℓ window survives the writer path. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +def _roundtrip(s, tmp_path, name): + p = tmp_path / f"{name}.sacc" + sio.save(s, str(p), type="mock") + return sio.load(str(p)) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +# --------------------------------------------------------------------------- # +# Per-writer parts +# --------------------------------------------------------------------------- # +def test_xi_to_sacc_reporting(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = sw.xi_to_sacc( + {0: _nz()}, META, theta, xip, xim, grid="reporting", theta_nom=theta * 1.01 + ) + s2 = _roundtrip(s, tmp_path, "xic") + th, p, m = sio.get_xi(s2, (0, 0), grid="reporting") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) and np.array_equal(m, xim) + assert s2.covariance is None # reporting part has no cov until assembly + + +def test_xi_to_sacc_integration_diagonal(tmp_path): + theta = np.geomspace(0.5, 300.0, 30) + xip, xim = np.arange(30) * 1e-5, np.arange(30) * 2e-5 + varxip, varxim = np.arange(1, 31) * 1e-12, np.arange(1, 31) * 2e-12 + s = sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + xip, + xim, + grid="integration", + variances=np.concatenate([varxip, varxim]), + ) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "xif") + th, p, _ = sio.get_xi(s2, (0, 0), grid="integration") + assert np.array_equal(th, theta) and np.array_equal(p, xip) + assert np.array_equal( + np.diag(s2.covariance.dense), np.concatenate([varxip, varxim]) + ) + + +def test_pseudo_cl_to_sacc_window_and_rows(tmp_path): + ell = np.array([30.0, 60.0, 90.0, 120.0]) + nbp = len(ell) + # NaMaster (4, nbp): EE, EB, BE, BB. + cl_all = np.vstack( + [ + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, + np.zeros(nbp), + np.arange(nbp) * 3e-9, + ] + ) + + class _Wsp: + """Stand-in workspace: (n_cl_out, nbp, n_cl_in, nell) window array.""" + + def __init__(self, nbp, nell): + w = np.zeros((4, nbp, 4, nell)) + col = np.zeros((nbp, nell)) + for b in range(nbp): + col[b, b * 3 : b * 3 + 3] = 1.0 + for out in range(4): + w[out, :, out, :] = col + self._w = w + + def get_bandpower_windows(self): + return self._w + + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Wsp(nbp, 24)) + s2 = _roundtrip(s, tmp_path, "cl") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) + assert np.array_equal(ee, cl_all[0]) # EE row + assert np.array_equal(bb, cl_all[3]) # BB row (index 3, not 2=BE) + assert np.array_equal(eb, cl_all[1]) # EB row + assert window.weight.shape == (24, nbp) + + +def _paper_pseudo_cl_reader(): + # A paper script, not a package module: load it by path. + path = Path(__file__).resolve().parents[3] / "papers/bmodes/scripts/pseudo_cl_io.py" + spec = importlib.util.spec_from_file_location("pseudo_cl_io", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.load_pseudo_cl_data + + +def test_paper_pseudo_cl_reader(tmp_path): + """The B-mode paper's reader maps NaMaster's EE/EB/BE/BB rows by name.""" + ell = np.array([30.0, 60.0, 90.0]) + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + + class _Workspace: + def get_bandpower_windows(self): + window = np.zeros((4, 3, 4, 12)) + for component in range(4): + for band in range(3): + window[component, band, component, band * 4 : (band + 1) * 4] = 1.0 + return window + + part = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Workspace()) + path = tmp_path / "pseudo_cl.sacc" + sio.save(part, str(path), type="mock") + + data = _paper_pseudo_cl_reader()(path) + assert np.array_equal(data["ELL"], ell) + assert np.array_equal(data["EE"], cl_all[0]) + assert np.array_equal(data["EB"], cl_all[1]) + assert np.array_equal(data["BB"], cl_all[3]) + + +def test_pseudo_cl_to_sacc_real_namaster(tmp_path): + """A real small-nside NaMaster workspace's window survives the writer.""" + pytest.importorskip("pymaster") + from sp_validation.pseudo_cl import get_pseudo_cls_map + + nside = 32 + mask = np.ones(12 * nside**2) + rng = np.random.default_rng(0) + shear = ( + rng.normal(size=12 * nside**2) + 1j * rng.normal(size=12 * nside**2) + ) * 1e-2 + ell_eff, cl_all, wsp = get_pseudo_cls_map(shear, mask, nside, "linear", ell_step=8) + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp) + s2 = _roundtrip(s, tmp_path, "clreal") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell_eff) + assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) + # window columns correspond to the bandpowers, one per ell_eff + assert window.weight.shape[1] == len(ell_eff) + + +def test_cosebis_to_sacc(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + result = {"En": En, "Bn": Bn, "cov": _spd(20, 7)} + s = sw.cosebis_to_sacc({0: _nz()}, META, result, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "co") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) and np.array_equal(B, Bn) + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.array_equal(s2.covariance.dense, result["cov"]) + + +def test_pure_eb_to_sacc(tmp_path): + theta = _theta() + eb = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + cov = _spd(6 * len(theta), 9) + s = sw.pure_eb_to_sacc({0: _nz()}, META, theta, eb, covariance=cov) + s2 = _roundtrip(s, tmp_path, "eb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], eb[key]) + assert np.array_equal(s2.covariance.dense, cov) + + +def _rho_tau_tables(nth=6, seed=0): + rng = np.random.default_rng(seed) + theta = _theta(nth) + rho = {"theta": theta} + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + tau = {"theta": theta} + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + return rho, tau, theta + + +def test_rho_tau_to_sacc_diagonal(tmp_path): + rho, tau, theta = _rho_tau_tables() + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau) + s2 = _roundtrip(s, tmp_path, "rt") + for k in sw.RHO_K: + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, rho[f"rho_{k}_p"]) + assert np.array_equal(m, rho[f"rho_{k}_m"]) + for k in sw.TAU_K: + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(p, tau[f"tau_{k}_p"]) + assert type(s2.covariance).__name__ == "DiagonalCovariance" + + +def test_rho_tau_to_sacc_tau_theory_block(tmp_path): + """The (3·nbin) plus-only CovTauTh block scatters into the τ-plus rows/cols; + τ-minus keeps a vartau diagonal, and cross plus↔minus stays zero.""" + rho, tau, theta = _rho_tau_tables() + nbin = len(theta) + n_plus = len(sw.TAU_K) * nbin # τ-plus points (k-major, one component per k) + tau_cov_th = _spd(n_plus, 11) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=tau_cov_th) + assert type(s.covariance).__name__ == "FullCovariance" + tr = ("source_0", sio.PSF_TRACER) + tau_plus = np.concatenate( + [s.indices(sio.TAU_PLUS.format(k=k), tr) for k in sw.TAU_K] + ) + tau_minus = np.concatenate( + [s.indices(sio.TAU_MINUS.format(k=k), tr) for k in sw.TAU_K] + ) + s2 = _roundtrip(s, tmp_path, "rttau") + dense = s2.covariance.dense + # τ-plus sub-block equals the supplied theory covariance (scatter is correct). + assert np.allclose(dense[np.ix_(tau_plus, tau_plus)], tau_cov_th) + # τ-minus is diagonal from vartau; plus↔minus cross is zero. + tau_minus_var = np.concatenate([np.asarray(tau[f"vartau_{k}_m"]) for k in sw.TAU_K]) + assert np.allclose(np.diag(dense[np.ix_(tau_minus, tau_minus)]), tau_minus_var) + assert np.allclose(dense[np.ix_(tau_plus, tau_minus)], 0.0) + + +def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): + rho, tau, _ = _rho_tau_tables() + with pytest.raises(ValueError, match="tau_cov_th shape"): + sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=_spd(3, 1)) + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _make_parts(nz): + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + xi.add_covariance(_spd(len(xi.mean), 1)) + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc(nz, META, ell, cl_all, _Wsp(), covariance=_spd(9, 2)) + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + return [xi, cl, co] + + +def test_assemble_analysis_sacc_block_diagonal_covariance(tmp_path): + nz = {0: _nz()} + parts = _make_parts(nz) + s = sw.assemble_analysis_sacc(parts) + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + # every point covered; blocks placed and cross-blocks zero + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert len(xi_idx) + len(cl_idx) + len(co_idx) == len(s.mean) + dense = s.covariance.dense + assert np.array_equal(dense[np.ix_(xi_idx, xi_idx)], parts[0].covariance.dense) + assert np.array_equal(dense[np.ix_(cl_idx, cl_idx)], parts[1].covariance.dense) + assert np.array_equal(dense[np.ix_(co_idx, co_idx)], parts[2].covariance.dense) + assert np.array_equal( + dense[np.ix_(xi_idx, cl_idx)], np.zeros((len(xi_idx), len(cl_idx))) + ) + # round-trips + s2 = _roundtrip(s, tmp_path, "analysis") + assert type(s2.covariance).__name__ == "BlockDiagonalCovariance" + assert np.allclose(s2.covariance.dense, s.covariance.dense) + + +def test_assemble_analysis_sacc_requires_covariance(): + nz = {0: _nz()} + parts = _make_parts(nz) + parts.append( + sw.xi_to_sacc( + nz, + META, + _theta(), + np.arange(6) * 1e-5, + np.arange(6) * 2e-5, + grid="reporting", + ) + ) # no covariance + with pytest.raises(ValueError, match="own covariance block"): + sw.assemble_analysis_sacc(parts) + + +def test_assemble_from_reloaded_parts(tmp_path): + """Parts written to disk then reloaded assemble identically (the DAG path).""" + nz = {0: _nz()} + parts = _make_parts(nz) + reloaded = [] + for i, part in enumerate(parts): + sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") + reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) + s = sw.assemble_analysis_sacc(reloaded) + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + +def test_xi_part_carries_a_dense_covariance(tmp_path): + """A grid measured with patches puts its jackknife block in the part.""" + theta = _theta() + cov = _spd(2 * len(theta), 41) + s = sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + np.arange(6) * 1e-5, + np.arange(6) * 2e-5, + grid="cosebis", + covariance=cov, + ) + s2 = _roundtrip(s, tmp_path, "xi_cov") + assert np.allclose(s2.covariance.dense, cov) + + +def test_xi_part_rejects_two_covariances(): + """Dense block and variances are alternatives, not a merge.""" + theta = _theta() + with pytest.raises(ValueError, match="not both"): + sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + np.zeros(6), + np.zeros(6), + grid="reporting", + covariance=_spd(12, 42), + variances=np.ones(12), + ) diff --git a/src/sp_validation/tests/test_xi_grids.py b/src/sp_validation/tests/test_xi_grids.py new file mode 100644 index 00000000..c35a2e3f --- /dev/null +++ b/src/sp_validation/tests/test_xi_grids.py @@ -0,0 +1,109 @@ +"""Tests for the ξ± grid table in ``workflow/common.py``. + +The table names the files the ``xi`` rule writes and the ones every consumer +asks for, so producer and consumer agree only if the tag is built from +canonical values. These tests pin that canonicalisation and the grid lookup. +""" + +import importlib.util +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.fast + + +def _load_common(): + root = next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + path = root / "workflow" / "common.py" + spec = importlib.util.spec_from_file_location("wf_common_grids", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _load_common() + +# A cosmo_val block as YAML delivers it: integer-valued separations stay ints. +CONFIG = { + "cosmo_val": { + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + "npatch": 100, + "integration": {"min_sep": 0.08, "max_sep": 300, "nbins": 1000}, + "cosebis": { + "min_sep_int": 0.9, + "max_sep_int": 300, + "nbins_int": 1000, + "npatch": 100, + }, + } +} +FIDUCIAL = { + "min_sep": 1.0, + "max_sep": 250.0, + "nbins": 20, + "npatch": 1, + "min_sep_int": 0.5, + "max_sep_int": 300, + "nbins_int": 1000, +} + + +def test_tag_is_built_from_canonical_values(): + """An integer YAML separation still names the file as a float. + + run_2pcf coerces separations with float() before TreeCorr writes, so a + `max_sep: 300` that reached the tag as "300" would have the consumer ask + for a path the producer never writes. + """ + grids = common.xi_grids(CONFIG, FIDUCIAL) + assert common.grid_binning(grids["integration"]).endswith( + "minsep=0.08_maxsep=300.0_nbins=1000_npatch=1" + ) + assert ( + common.grid_binning(grids["cosebis"]) + == "minsep=0.9_maxsep=300.0_nbins=1000_npatch=100" + ) + # Counts stay integers, so no "nbins=1000.0" creeps into a name. + assert "nbins=1000_" in common.grid_binning(grids["cosebis"]) + + +def test_grid_lookup_round_trips_through_the_tag(): + """Every grid's own binning resolves back to that grid. + + This is the producer/consumer contract: the rule resolves a job's grid from + the wildcards its filename bound. + """ + grids = common.xi_grids(CONFIG, FIDUCIAL) + for name, grid in grids.items(): + binning = {key: grid[key] for key in common.XI_KEYS} + assert common.grid_of(grids, binning) == name + # Wildcards arrive as strings; the comparison is numeric. + assert common.grid_of(grids, {k: str(v) for k, v in binning.items()}) == name + + +def test_covariance_mode_follows_the_patches(): + """Patched grids get a jackknife block, unpatched ones none.""" + grids = common.xi_grids(CONFIG, FIDUCIAL) + assert grids["reporting"]["cov"] == "jackknife" + assert grids["cosebis"]["cov"] == "jackknife" + assert grids["integration"]["cov"] == "none" + + +def test_unnamed_binning_is_a_reporting_measurement(): + """The paper's convergence-check binning belongs to no named grid.""" + grids = common.xi_grids(CONFIG, FIDUCIAL) + stray = {"min_sep": 1.0, "max_sep": 250.0, "nbins": 10000, "npatch": 1} + assert common.grid_of(grids, stray) == "reporting" + + +def test_workflow_without_cosmo_val_falls_back_to_fiducial(): + """papers/bmodes carries no cosmo_val block; its grids come from FIDUCIAL.""" + grids = common.xi_grids({}, FIDUCIAL) + assert grids["reporting"]["npatch"] == 1 + assert grids["integration"]["min_sep"] == 0.5 + assert "cosebis" not in grids diff --git a/workflow/common.py b/workflow/common.py index 32a76b83..4391ddbf 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -44,6 +44,9 @@ ) ) CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" +# "blind" is the glass-mock A/B/C realisation convention, NOT Smokescreen +# blinding (a separate axis: the concealed=True SACC stamp). The name is baked +# into on-disk filenames we do not own (e.g. nz_{version}_{A|B|C}.txt). BLINDS = ["A", "B", "C"] BLOCK_PAIRS = [("++", "1"), ("--", "2"), ("+-", "3")] @@ -263,22 +266,122 @@ def covariance_path( return str(COSMO_INFERENCE / f"data/covariance/{base}/{base}{suffix}") +def base_version(version): + """Strip the derived-catalogue suffixes to the base catalogue version. + + The `_leak_corr` / `_ecut{N}` variants share their parent's n(z) and + `cov_th` survey parameters, so lookups keyed on either must strip both. + """ + return re.sub(r"_ecut\d+", "", re.sub(r"_leak_corr$", "", version)) + + def build_redshift_path(version, blind): """Construct n(z) filepath for given catalog version and blind.""" - base_version = re.sub(r"_leak_corr$", "", version) - base_version = re.sub(r"_ecut\d+", "", base_version) - if "v1.4.11" in base_version: - base_version = "SP_v1.4.6" - version_dir = base_version.replace("SP_", "") + base = base_version(version) + if "v1.4.11" in base: + base = "SP_v1.4.6" + version_dir = base.replace("SP_", "") + return f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base}_{blind}.txt" + + +# --------------------------------------------------------------------------- +# ξ± angular grids +# --------------------------------------------------------------------------- +# A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, +# nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine +# one the B-mode integrals run over, `cosebis` the fine patched grid COSEBIs +# propagates its covariance from. cov is "jackknife" (dense, from the patches), +# "diagonal" (TreeCorr varxip/varxim) or "none". +XI_KEYS = ( + "min_sep", + "max_sep", + "nbins", + "npatch", +) # the binning; cov is not in the name + + +def xi_grids(config, fiducial): + """The named ξ± grids of a workflow, canonicalised. + + Workflows carrying no cosmo_val block (e.g. papers/bmodes) fall back to + ``fiducial``. Values are coerced here — separations to float, counts to int + — so the tag the table stamps into a filename is the one the measurement + writes: the separations pass through float() on the way to TreeCorr, so a + YAML ``300`` must become ``300.0`` before it names a file, or producer and + consumer ask for different paths. + """ + cv = config.get("cosmo_val", {}) + grids = { + "reporting": ( + { + "min_sep": cv["theta_min"], + "max_sep": cv["theta_max"], + "nbins": cv["nbins"], + "npatch": cv["npatch"], + } + if cv + else {k: fiducial[k] for k in XI_KEYS} + ), + "integration": dict( + cv.get("integration") + or { + "min_sep": fiducial["min_sep_int"], + "max_sep": fiducial["max_sep_int"], + "nbins": fiducial["nbins_int"], + } + ), + } + grids["integration"].setdefault("npatch", 1) + cb = cv.get("cosebis") + if cb: + grids["cosebis"] = { + "min_sep": cb["min_sep_int"], + "max_sep": cb["max_sep_int"], + "nbins": cb["nbins_int"], + "npatch": cb["npatch"], + } + for grid in grids.values(): + for key in ("min_sep", "max_sep"): + grid[key] = float(grid[key]) + for key in ("nbins", "npatch"): + grid[key] = int(grid[key]) + # A jackknife estimate needs patches; at npatch=1 TreeCorr's var_method + # is "shot" and the diagonal is all it can offer. + grid.setdefault("cov", "jackknife" if grid["npatch"] > 1 else "none") + return grids + + +def grid_binning(grid): + """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of one grid.""" return ( - f"/n17data/sguerrini/UNIONS/WL/nz/{version_dir}/nz_{base_version}_{blind}.txt" + f"minsep={grid['min_sep']}_maxsep={grid['max_sep']}" + f"_nbins={grid['nbins']}_npatch={grid['npatch']}" ) +def grid_of(grids, binning): + """Name of the grid a binning belongs to, compared numerically. + + A "300" wildcard matches a 300.0 grid value. Binnings matching no named + grid (e.g. papers/bmodes' nbins=10000 convergence check) are reporting-style + measurements. + """ + key = tuple(float(binning[k]) for k in XI_KEYS) + for name, grid in grids.items(): + if tuple(float(grid[k]) for k in XI_KEYS) == key: + return name + return "reporting" + + +def pseudo_cl_tag(config): + """Fiducial harmonic-binning tag stamped into pseudo-Cl filenames.""" + fiducial = config["harmonic"]["fiducial"] + return f"blind={fiducial['blind']}_{fiducial['binning']}_nbins={fiducial['nbins']}" + + def get_shear_catalog(wildcards): """Resolve shear catalog path from config for a given version.""" - base_version = wildcards.version.replace("_leak_corr", "") - cat_config = CATALOG_CONFIG[base_version] + cat_config = CATALOG_CONFIG[wildcards.version.replace("_leak_corr", "")] shear_path = cat_config["shear"]["path"] if shear_path.startswith("/"): return shear_path diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 704b4fb0..87138bc0 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -3,29 +3,33 @@ # The original cosmo_val/run_cosmo_val.py was one linear driver that built a # single in-memory `cv` (CosmologyValidation) and called ~13 cv.() # diagnostics in sequence, linked only by lazy properties on that object. Here -# each diagnostic is a rule, and the rules are linked by the *real* data -# products each method writes under COSMO_VAL (= cosmo_val/output): +# each diagnostic is a rule, and the rules are linked by the SACC parts and +# products they write under COSMO_VAL (= cosmo_val/output): # -# rho/tau FITS ──┬─→ rho/tau plots -# ├─→ rho_tau_fits (PSF-error MCMC) -# └─────────────────────────────┐ -# additive bias ──→ xi (2pcf) ──┬─→ 2pcf plot │ -# ├─→ ratio_xi_sys_xi ←┘ (also needs xi_psf_sys) -# ├─→ pure_eb (npz) ─┐ -# └─→ cosebis (npz) ─┤ -# pseudo_cl FITS ──────────────────────────────────┼─→ summarize_bmodes -# ┘ +# catalogue ──→ xi (one job per grid: reporting, integration, cosebis) +# │ +# ├─ reporting part ──┬─→ pure_eb (part, npz, figures) +# ├─ integration part ┘ │ +# ├─ cosebis part ─────→ cosebis (part, npz, figures) +# └─ reporting .txt ──→ 2pcf plot, ratio_xi_sys_xi +# catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures +# CosmoCov ──→ covariance ─────────┤ +# rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products) +# └─→ assemble_sacc ──→ {version}.sacc # -# Granularity decision: methods that write durable data products -# (calculate_rho_tau_stats, calculate_2pcf, calculate_pseudo_cl, plot_pure_eb, -# plot_cosebis) own a compute rule keyed on those files. Methods that only -# emit figures, or whose figure paths derive from internal handler state -# (rho/tau plots, rho_tau_fits, objectwise leakage, 2pcf overlay), declare a -# sentinel under COSMO_VAL/snakemake_sentinels so they stay DAG-trackable. -# Lazy cv state that the original code never persists (c1/c2, xi_psf_sys) is -# either materialized to a small JSON (additive bias) or recomputed in the one -# rule that needs it (xi_psf_sys in ratio_xi_sys_xi) — recompute is cheap next -# to the science it depends on. See workflow/scripts/cv_runner.py. +# The B-mode rules are ingests: pure_eb, cosebis, the pseudo-Cl figures and the +# summary all work from the parts and the covariance inputs, never from a +# catalogue, so a blinded part keeps everything downstream blinded. The +# analytic covariances (CosmoCov ξ±, NaMaster pseudo-Cℓ) are what assembly puts +# in the terminal file, replacing the estimates a part was born with. +# +# Methods that only emit figures, or whose figure paths derive from internal +# handler state (rho/tau plots, rho_tau_fits, objectwise leakage, 2pcf +# overlay), declare a sentinel under COSMO_VAL/snakemake_sentinels so they stay +# DAG-trackable. Lazy cv state the original code never persists (c1/c2, +# xi_psf_sys) is either materialized to a small JSON (additive bias) or +# recomputed in the one rule that needs it — recompute is cheap next to the +# science it depends on. See workflow/scripts/cv_runner.py. CV = config["cosmo_val"] CV_VERSIONS = config["versions"] @@ -52,13 +56,7 @@ def cv_xi_txt(version): Mirrors the out_fname f-string in cosmo_val.calculate_2pcf: {ver}_xi_minsep=..._maxsep=..._nbins=..._npatch=...txt """ - return str( - COSMO_VAL - / ( - f"{version}_xi_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" - f"_nbins={CV['nbins']}_npatch={CV['npatch']}.txt" - ) - ) + return str(COSMO_VAL / f"{version}_xi_{xi_binning('reporting')}.txt") def cv_rho_stats(version): @@ -73,39 +71,140 @@ def cv_tau_stats(version): ) -def cv_pure_eb_npz(version): - eb = CV["pure_eb"] +def _pure_eb_stub(version): + """Shared stem of the pure-E/B diagnostic products (npz + figures).""" + eb = CV["integration"] return str( COSMO_VAL / ( f"{version}_eb_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" - f"_nbins={CV['nbins']}_minsepint={eb['min_sep_int']}" - f"_maxsepint={eb['max_sep_int']}_nbinsint={eb['nbins_int']}" - f"_npatch={CV['npatch']}_varmethod=jackknife_data.npz" + f"_nbins={CV['nbins']}_minsepint={eb['min_sep']}" + f"_maxsepint={eb['max_sep']}_nbinsint={eb['nbins']}" + f"_npatch={CV['npatch']}_varmethod=semi-analytic" ) ) -def cv_cosebis_npz(version): +def cv_pure_eb_npz(version): + """Pure-E/B data vectors + covariance .npz.""" + return _pure_eb_stub(version) + "_data.npz" + + +def cv_pure_eb_figures(version): + """The pure-E/B companion figures, by output key.""" + stub = _pure_eb_stub(version) + return { + "figure_integration_vs_reporting": f"{stub}_integration_vs_reporting.png", + "figure_xis": f"{stub}_xis.png", + "figure_ptes": f"{stub}_ptes.png", + "figure_covariance": f"{stub}_covariance.png", + } + + +def cv_xi_cov_integration(version): + """CosmoCov gaussian ξ± covariance on the integration grid. + + The covariance model the pure-E/B Monte Carlo draws from; gaussian because + the draws only need the scatter a Gaussian field would give. + """ + integ = CV["integration"] + return covariance_path( + version, + FIDUCIAL["blind"], + gaussian="g", + min_sep=integ["min_sep"], + max_sep=integ["max_sep"], + nbins=integ["nbins"], + mask_suffix=DEFAULT_MASK_SUFFIX, + ) + + +def _cosebis_stub(version): + """Shared stem of the COSEBIs diagnostic products (npz + figures). + + varmethod names where the covariance came from, and these products are the + propagated one — which also keeps them clear of the paths plot_cosebis + builds for its own byproducts, so nothing overwrites a declared output. + """ cb = CV["cosebis"] fsc = CV["fiducial_scale_cut"] - # Mirror calculate/plot_cosebis out_stub (cosmo_val.py): a distinct schema - # from pure_eb — _cosebis_ prefix, integration nbins, plus _nmodes= and - # _scalecut= segments. Must match save_cosebis_results exactly or - # verify_outputs raises and cv_summarize_bmodes deadlocks on this input. return str( COSMO_VAL / ( f"{version}_cosebis_minsep={cb['min_sep_int']}" f"_maxsep={cb['max_sep_int']}_nbins={cb['nbins_int']}" - f"_npatch={cb['npatch']}_varmethod=jackknife_nmodes={cb['nmodes']}" - f"_scalecut={fsc[0]}-{fsc[1]}_data.npz" + f"_npatch={cb['npatch']}_varmethod=propagated_nmodes={cb['nmodes']}" + f"_scalecut={fsc[0]}-{fsc[1]}" ) ) -def cv_pseudo_cl_fits(version): - return str(COSMO_VAL / f"pseudo_cl_{version}.fits") +def cv_cosebis_npz(version): + """COSEBIs multi-cut diagnostic .npz (the PTE scan).""" + return _cosebis_stub(version) + "_data.npz" + + +def cv_cosebis_figures(version): + """The COSEBIs companion figures, by output key.""" + stub = _cosebis_stub(version) + return { + "figure_modes": f"{stub}_cosebis.png", + "figure_covariance": f"{stub}_covariance.png", + "figure_scalecut_ptes": f"{stub}_scalecut_ptes.png", + } + + +_PSEUDO_CL_TAG = pseudo_cl_tag(config) + + +def cv_pseudo_cl_analysis_sacc(version): + """Tagged pseudo-Cl SACC part: the harmonic block of the analysis file.""" + return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") + + +def cv_pseudo_cl_cov(version): + """NaMaster pseudo-Cl covariance FITS (COVAR_EE_EE/BB_BB/EB_EB extensions).""" + return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}.fits") + + +def cv_xi_cov(version): + """CosmoCov-processed ξ± covariance, on the reporting grid's own binning.""" + return covariance_path( + version, + FIDUCIAL["blind"], + gaussian="ng", + min_sep=CV["theta_min"], + max_sep=CV["theta_max"], + nbins=CV["nbins"], + mask_suffix=DEFAULT_MASK_SUFFIX, + ) + + +def cv_cosebis_sacc(version): + """COSEBIs SACC part, at the fiducial scale cut.""" + return str(COSMO_VAL / f"{version}_cosebis.sacc") + + +def cv_pure_eb_sacc(version): + """Pure-E/B SACC part.""" + return str(COSMO_VAL / f"{version}_pure_eb.sacc") + + +def cv_rho_tau_sacc(version): + """ρ/τ SACC part.""" + return str( + COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}.sacc" + ) + + +def cv_xi_sacc(version, grid): + """ξ± SACC part for a version on a named grid, named by that grid's binning.""" + return str(COSMO_VAL / f"{version}_xi_{xi_binning(grid)}.sacc") + + +def cv_analysis_sacc(version): + """Terminal assembled analysis file {version}.sacc.""" + return str(COSMO_VAL / f"{version}.sacc") # Common params block shared by every cosmo_val rule: the cv constructor kwargs @@ -274,18 +373,32 @@ rule cv_ratio_xi_sys_xi: # Harmonic-space pseudo-Cl # --------------------------------------------------------------------------- -rule cv_pseudo_cl: - """Pseudo-Cl E/B spectra for all versions (NaMaster).""" +def cv_pseudo_cl_figures(): + """The pseudo-Cl figures, by output key (one per spectrum, all versions).""" + return { + f"figure_{name}": str(COSMO_VAL / f"cell_{name}.png") + for name in ("ee", "eb", "bb") + } + + +rule cv_plot_pseudo_cl: + """The EE/EB/BB pseudo-Cl figures, from the analysis parts.""" + input: + pseudo_cl=[cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS], + pseudo_cl_cov=[cv_pseudo_cl_cov(v) for v in CV_VERSIONS], output: - pseudo_cl=[cv_pseudo_cl_fits(v) for v in CV_VERSIONS], + **cv_pseudo_cl_figures(), params: - **cv_params(), - threads: 12 + versions=CV_VERSIONS, + # Style is per catalogue, so the derived variants take their parent's. + markers=[CATALOG_CONFIG[base_version(v)]["marker"] for v in CV_VERSIONS], + colours=[CATALOG_CONFIG[base_version(v)]["colour"] for v in CV_VERSIONS], + rundir=CV_RUNDIR, resources: - mem_mb=32000, - runtime=180, + mem_mb=8000, + runtime=20, script: - "../scripts/cv_pseudo_cl.py" + "../scripts/cv_plot_pseudo_cl.py" # --------------------------------------------------------------------------- @@ -293,18 +406,27 @@ rule cv_pseudo_cl: # --------------------------------------------------------------------------- rule cv_pure_eb: - """Pure E/B-mode decomposition for one version (config-space).""" + """Pure E/B-mode decomposition for one version, from its ξ± parts. + + The modes come from the two parts; the covariance is Monte Carlo from the + integration-grid covariance model, so no patched estimator run is involved. + """ input: - xi=lambda w: cv_xi_txt(w.version), + xi_reporting=lambda w: cv_xi_sacc(w.version, "reporting"), + xi_integration=lambda w: cv_xi_sacc(w.version, "integration"), + cov_integration=lambda w: cv_xi_cov_integration(w.version), output: npz=cv_pure_eb_npz("{version}"), + sacc=cv_pure_eb_sacc("{version}"), + **cv_pure_eb_figures("{version}"), params: version="{version}", - min_sep_int=CV["pure_eb"]["min_sep_int"], - max_sep_int=CV["pure_eb"]["max_sep_int"], - nbins_int=CV["pure_eb"]["nbins_int"], + min_sep=CV["theta_min"], + max_sep=CV["theta_max"], + nbins=CV["nbins"], + n_samples=CV.get("n_mc_samples", 1000), + cosmo_params=CV["cosmo_params"], fiducial_scale_cut=CV["fiducial_scale_cut"], - cv_init=lambda w: cv_init_params(config, version_list=[w.version]), rundir=CV_RUNDIR, threads: 24 resources: @@ -315,21 +437,25 @@ rule cv_pure_eb: rule cv_cosebis: - """COSEBIs E/B decomposition for one version (config-space, fine binning).""" + """COSEBIs E/B decomposition for one version, from its ξ± part. + + Values, covariance and PTEs all come from the part: the COSEBIs covariance + is the part's ξ± covariance through the same kernel as the modes. + """ input: - xi=lambda w: cv_xi_txt(w.version), + xi=lambda w: cv_xi_sacc(w.version, "cosebis"), output: npz=cv_cosebis_npz("{version}"), + sacc=cv_cosebis_sacc("{version}"), + **cv_cosebis_figures("{version}"), params: version="{version}", - min_sep_int=CV["cosebis"]["min_sep_int"], - max_sep_int=CV["cosebis"]["max_sep_int"], - nbins_int=CV["cosebis"]["nbins_int"], - npatch=CV["cosebis"]["npatch"], + min_sep=CV["cosebis"]["min_sep_int"], + max_sep=CV["cosebis"]["max_sep_int"], + nbins=CV["cosebis"]["nbins_int"], nmodes=CV["cosebis"]["nmodes"], scale_cuts=CV["cosebis"]["scale_cuts"], fiducial_scale_cut=CV["fiducial_scale_cut"], - cv_init=lambda w: cv_init_params(config, version_list=[w.version]), rundir=CV_RUNDIR, threads: 24 resources: @@ -345,32 +471,86 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [cv_pseudo_cl_fits(v) for v in CV_VERSIONS] + [cv_pseudo_cl_analysis_sacc(v) for v in CV_VERSIONS] + if CV.get("include_pseudo_cl", False) else [] + ), + pseudo_cl_cov=( + [cv_pseudo_cl_cov(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), output: summary_json=str(COSMO_VAL / "bmode_summary.json"), params: + versions=CV_VERSIONS, fiducial_scale_cut=CV["fiducial_scale_cut"], - pure_eb_min_sep_int=CV["pure_eb"]["min_sep_int"], - pure_eb_max_sep_int=CV["pure_eb"]["max_sep_int"], - pure_eb_nbins_int=CV["pure_eb"]["nbins_int"], - cosebis_min_sep_int=CV["cosebis"]["min_sep_int"], - cosebis_max_sep_int=CV["cosebis"]["max_sep_int"], - cosebis_nbins_int=CV["cosebis"]["nbins_int"], - cosebis_npatch=CV["cosebis"]["npatch"], - cosebis_nmodes=CV["cosebis"]["nmodes"], - cosebis_scale_cuts=CV["cosebis"]["scale_cuts"], + min_sep=CV["theta_min"], + max_sep=CV["theta_max"], + nbins=CV["nbins"], include_pseudo_cl=CV.get("include_pseudo_cl", False), - **cv_params(), - threads: 24 + rundir=CV_RUNDIR, resources: - mem_mb=48000, - runtime=600, + mem_mb=8000, + runtime=20, script: "../scripts/cv_summarize_bmodes.py" +# --------------------------------------------------------------------------- +# Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc +# --------------------------------------------------------------------------- +# The terminal file carries the analysis vector only. The integration-grid ξ± is +# deliberately not gathered: it stays a per-part intermediate. The two blocks +# born without a covariance (ξ± reporting, pseudo-Cℓ) get theirs injected from +# the covariance inputs below. + + +def cv_assemble_inputs(version): + """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. + + Each part's filename carries enough to bind its producing rule's wildcards. + """ + parts = dict( + xi_reporting=cv_xi_sacc(version, "reporting"), + xi_cov=cv_xi_cov(version), + cosebis=cv_cosebis_sacc(version), + pure_eb=cv_pure_eb_sacc(version), + rho_tau=cv_rho_tau_sacc(version), + ) + if CV.get("include_pseudo_cl", False): + parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) + parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) + return parts + + +rule assemble_sacc: + """Assemble the terminal {version}.sacc from the per-statistic SACC parts.""" + input: + unpack(lambda w: cv_assemble_inputs(w.version)), + output: + sacc=cv_analysis_sacc("{version}"), + params: + version="{version}", + type=CV.get("type", "data"), + # The statistics this rule wired, so a typo'd input keyword cannot + # silently drop one. + expected=lambda w: [ + k + for k in cv_assemble_inputs(w.version) + if k not in ("xi_cov", "pseudo_cl_cov") + ], + resources: + mem_mb=8000, + runtime=20, + script: + "../scripts/assemble_sacc.py" + + +rule assemble_sacc_all: + """Assemble the analysis SACC file for every version.""" + input: + [cv_analysis_sacc(v) for v in CV_VERSIONS], + + # --------------------------------------------------------------------------- # Aggregate target: the whole validation suite # --------------------------------------------------------------------------- @@ -392,3 +572,6 @@ rule cosmo_val_all: str(COSMO_VAL / "ratio_xi_sys_xi.png"), # B-modes str(COSMO_VAL / "bmode_summary.json"), + list(cv_pseudo_cl_figures().values()) if CV.get("include_pseudo_cl", False) else [], + # Terminal analysis file: the assembled {version}.sacc per version + [cv_analysis_sacc(v) for v in CV_VERSIONS], diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index b2df6c7c..1b6663b6 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -1,13 +1,34 @@ # Two-point data-vector rules: xi, rho/tau, and pseudo-Cl products. +# --------------------------------------------------------------------------- +# ξ± angular grids +# --------------------------------------------------------------------------- +# The table itself lives in common.py, where it can be built from a plain config +# dict and tested; these are the workflow's bindings to it. +XI_GRIDS = xi_grids(config, FIDUCIAL) + + +def xi_binning(grid): + """The `minsep=..._maxsep=..._nbins=..._npatch=...` tag of a named grid.""" + return grid_binning(XI_GRIDS[grid]) + + +def xi_grid_of(wildcards): + """Grid label for the binning a job was requested with.""" + return grid_of(XI_GRIDS, {key: getattr(wildcards, key) for key in XI_KEYS}) + rule xi: + """TreeCorr ξ±(θ) for one version on one angular grid. + + One rule for every grid: outputs are named by their binning, so a request + binds the wildcards and `xi_grid_of` resolves the grid label from them. + """ input: catalog=get_shear_catalog, output: - str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), + sacc=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -17,11 +38,14 @@ rule xi: npatch="{npatch}", cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - fits=False, + grid=lambda w: xi_grid_of(w), + cov=lambda w: XI_GRIDS[xi_grid_of(w)]["cov"], resources: - mem_mb=30000, + # The fine integration grid needs more memory and wall time than the + # ~20-bin reporting one; scale on nbins rather than splitting the rule. + mem_mb=lambda w: 40000 if int(w.nbins) > 100 else 30000, disk_mb=20000, - runtime=360, + runtime=lambda w: 600 if int(w.nbins) > 100 else 360, script: "../scripts/run_2pcf.py" @@ -30,6 +54,8 @@ rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Born-as-SACC ρ/τ part, written alongside the FITS. + rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: ver="{version}", @@ -54,9 +80,9 @@ wildcard_constraints: rule pseudo_cl: - """Generate pseudo-Cl data vector with configurable binning.""" + """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", params: @@ -108,7 +134,7 @@ rule pseudo_cl_all: """Generate pseudo-Cls for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc"), version=PSEUDO_CL_VERSIONS, ), @@ -126,7 +152,7 @@ rule pseudo_cl_fine_all: """Generate fine pseudo-Cls for COSEBIS.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.sacc"), version=config["versions"], blind=BLINDS, ), diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py new file mode 100644 index 00000000..50d1d62e --- /dev/null +++ b/workflow/scripts/assemble_sacc.py @@ -0,0 +1,203 @@ +"""Assemble the terminal ``{version}.sacc`` analysis file from per-statistic parts. + +Dual-mode: under Snakemake (``script:``) the injected ``snakemake`` object +supplies the inputs; as a standalone CLI the same assembly runs from flags. + +Each part is a single-statistic SACC; they load in CANONICAL order and are +rebuilt into one Sacc with a single ``BlockDiagonalCovariance``. + +Every part must carry a covariance block. ξ± reporting and pseudo-Cℓ take +theirs from the analytic inputs — the CosmoCov ``.txt`` (``--xi-cov``) and the +NaMaster covariance FITS (``--pseudo-cl-cov``) — which replace any estimate the +part was born with; the pseudo-Cℓ cross-spectrum blocks (EE↔BB, …) are dropped, +matching what the B-mode PTE reads today. +""" + +import argparse + +import numpy as np + +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc + +# NaMaster iNKA covariance FITS: per-spectrum HDU names, in SACC insertion order. +_CL_HDUS = ("COVAR_EE_EE", "COVAR_BB_BB", "COVAR_EB_EB") + +# Canonical part order — the order points are inserted in, which must match the +# covariance block order. Missing parts are simply skipped. +CANONICAL = ("xi_reporting", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") + + +def _pseudo_cl_cov_block(cov_fits): + """Block-diagonal ``[EE; BB; EB]`` from the NaMaster iNKA covariance FITS.""" + from astropy.io import fits + + with fits.open(cov_fits) as hdul: + missing = [name for name in _CL_HDUS if name not in {h.name for h in hdul}] + if missing: + raise ValueError(f"{cov_fits} lacks the pseudo-Cℓ cov HDUs {missing}") + blocks = [np.asarray(hdul[name].data, float) for name in _CL_HDUS] + n = blocks[0].shape[0] + full = np.zeros((3 * n, 3 * n)) + for i, block in enumerate(blocks): + full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block + return full + + +# The statistics whose analysis covariance is external, and the input each one +# takes it from. A part of one of these types may be born with an estimate of +# its own — the ξ± reporting part carries the jackknife it was measured with — +# but the analysis file takes the external one, always. +_INJECTED = {"xi_reporting": "xi_cov", "pseudo_cl": "pseudo_cl_cov"} + + +def _attach_cov(part, name, xi_cov, pseudo_cl_cov): + """Give ``part`` (mutated in place) the covariance the analysis file uses. + + For the two statistics with an external covariance the supplied block + replaces whatever the part was born with, loudly; every other part keeps + its own. Raises if the block a part needs was not supplied. + """ + if name not in _INJECTED: + if part.covariance is None: + raise ValueError( + f"the {name!r} part carries no covariance and none is injected " + "for it; its writer must attach one" + ) + return part + + supplied = xi_cov if name == "xi_reporting" else pseudo_cl_cov + if supplied is None: + raise ValueError( + f"the {name!r} part takes its analysis covariance from " + f"--{_INJECTED[name].replace('_', '-')}, which was not supplied" + ) + block = ( + np.loadtxt(supplied) + if name == "xi_reporting" + else _pseudo_cl_cov_block(supplied) + ) + if part.covariance is not None: + print( + f"{name}: replacing the part's own covariance with {supplied} " + "(the analysis covariance)" + ) + part.add_covariance(block, overwrite=True) + return part + + +def assemble_sacc( + version, + part_paths, + out_path, + *, + expected=None, + xi_cov=None, + pseudo_cl_cov=None, + allow_unblinded=False, +): + """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. + + Parameters + ---------- + version : str + Catalogue version, for error messages. + part_paths : dict + ``{statistic: path}`` with statistic in :data:`CANONICAL`. Only present + statistics are assembled; order is forced to canonical. + expected : sequence of str, optional + Statistics that must be present, from the caller's config toggles. A + typo'd input keyword would otherwise silently drop a statistic. + xi_cov, pseudo_cl_cov + Covariance sourcing — see the module docstring. + allow_unblinded : bool, optional + Passed to :func:`sacc_io.load` for every part; ``True`` only for mocks. + """ + if expected is not None: + unknown = [name for name in expected if name not in CANONICAL] + if unknown: + raise ValueError( + f"expected parts {unknown} are not assemblable statistics; " + f"valid names are {CANONICAL}" + ) + missing = [name for name in expected if not part_paths.get(name)] + if missing: + raise ValueError( + f"expected parts {missing} missing from part_paths for {version} " + f"(got {sorted(part_paths)}); a required statistic would be " + "silently dropped from the terminal analysis file" + ) + parts = [] + for name in CANONICAL: + path = part_paths.get(name) + if path is None: + continue + part = sacc_io.load(path, allow_unblinded=allow_unblinded) + parts.append(_attach_cov(part, name, xi_cov, pseudo_cl_cov)) + if not parts: + raise ValueError(f"no parts found for {version}: {part_paths}") + s = assemble_analysis_sacc(parts) + sacc_io.save(s, out_path, type=s.metadata["type"]) + print(f"Assembled {len(parts)} parts -> {out_path}") + return s + + +def _from_snakemake(smk): + p = smk.params + inp = smk.input + part_paths = { + name: getattr(inp, name) + for name in CANONICAL + if hasattr(inp, name) and getattr(inp, name) + } + assemble_sacc( + version=p["version"], + part_paths=part_paths, + out_path=str(smk.output[0]), + expected=list(p["expected"]), + xi_cov=getattr(inp, "xi_cov", None), + pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), + allow_unblinded=(p.get("type", "data") == "mock"), + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Assemble the terminal {version}.sacc from per-statistic parts." + ) + ap.add_argument("--version", required=True, help="Catalogue version") + ap.add_argument("--out", required=True, help="Output {version}.sacc path") + ap.add_argument( + "--type", + choices=("data", "mock"), + default="data", + help="Run type. 'mock' reads parts freely; 'data' fails closed on " + "unblinded parts (only concealed/blinded parts load).", + ) + for name in CANONICAL: + ap.add_argument( + f"--{name.replace('_', '-')}", default=None, help=f"{name} part" + ) + ap.add_argument("--xi-cov", default=None, help="CosmoCov ξ covariance .txt") + ap.add_argument( + "--pseudo-cl-cov", default=None, help="NaMaster pseudo-Cℓ covariance FITS" + ) + a = ap.parse_args(argv) + part_paths = {name: getattr(a, name) for name in CANONICAL if getattr(a, name)} + assemble_sacc( + version=a.version, + part_paths=part_paths, + out_path=a.out, + xi_cov=a.xi_cov, + pseudo_cl_cov=a.pseudo_cl_cov, + allow_unblinded=(a.type == "mock"), + ) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index c1ca8275..c9241eaa 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,25 +1,70 @@ """Rule cv_cosebis: COSEBIs E/B decomposition for one version. -Compute + plot rule (per version). plot_cosebis calls calculate_cosebis over a -fine integration binning (the 2000-bin TreeCorr is the dominant cost) and -evaluates the configured scale cuts. Writes the {version}_eb_..._data.npz -COSEBIs data product (declared output) plus figures, and the per-version -COSEBIs PTE that cv_summarize_bmodes collects. +A consumer of the ξ± part alone — values, covariance, PTEs and figures all +derive from it, so nothing here touches a catalogue. The part's ξ± covariance +goes through the same linear kernel as the modes to give the COSEBIs +covariance; its ``npatch`` metadata sets the Hartlap debiasing. """ -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +from cv_runner import _unbuffer_streams, verify_outputs + +from sp_validation import sacc_io +from sp_validation.b_modes import ( + cosebis_scan_from_xi, + find_conservative_scale_cut_key, + log_bin_edges, + plot_cosebis_covariance_matrix, + plot_cosebis_modes, + plot_cosebis_scale_cut_heatmap, + save_cosebis_results, +) +from sp_validation.cosmo_val.sacc_writers import cosebis_to_sacc _unbuffer_streams() -cv = make_cv(snakemake) p = snakemake.params -cv.plot_cosebis( - version=p["version"], - min_sep_int=p["min_sep_int"], - max_sep_int=p["max_sep_int"], - nbins_int=p["nbins_int"], - npatch=p["npatch"], +version = p["version"] +fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) + +part = sacc_io.load(snakemake.input["xi"]) +theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="cosebis") +edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) + +results = cosebis_scan_from_xi( + theta, + xip, + xim, + part.covariance.dense, + *edges, nmodes=p["nmodes"], scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], - fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), + npatch=part.metadata["npatch"], +) + +fiducial_key = find_conservative_scale_cut_key(results, fiducial_scale_cut) +fiducial = results[fiducial_key] + +plot_cosebis_modes( + fiducial, + version, + snakemake.output["figure_modes"], + fiducial_scale_cut=fiducial_scale_cut, +) +plot_cosebis_covariance_matrix( + fiducial, version, "jackknife", snakemake.output["figure_covariance"] +) +plot_cosebis_scale_cut_heatmap( + results, + edges, + version, + snakemake.output["figure_scalecut_ptes"], + fiducial_scale_cut=fiducial_scale_cut, ) + +save_cosebis_results(results, snakemake.output["npz"], fiducial_scale_cut) + +# The part inherits the ξ± part's provenance; `type` is re-stamped on save. +metadata = {k: v for k, v in part.metadata.items() if k != "type"} +s = cosebis_to_sacc({0: sacc_io.get_nz(part, 0)}, metadata, fiducial, fiducial_key) +sacc_io.save(s, snakemake.output["sacc"], type="data") + verify_outputs(snakemake) diff --git a/workflow/scripts/cv_plot_pseudo_cl.py b/workflow/scripts/cv_plot_pseudo_cl.py new file mode 100644 index 00000000..f9aeeda9 --- /dev/null +++ b/workflow/scripts/cv_plot_pseudo_cl.py @@ -0,0 +1,39 @@ +"""Rule cv_plot_pseudo_cl: the EE/EB/BB pseudo-Cl figures. + +Plot-only, and an ingest like the other B-mode rules: the spectra come from the +analysis pseudo-Cl parts and their NaMaster covariances, the same pair the +summary and the terminal file are built from, so the figures cannot show +something the data products do not. +""" + +import numpy as np +from astropy.io import fits +from cv_runner import _unbuffer_streams, verify_outputs + +from sp_validation import sacc_io +from sp_validation.cosmo_val.pseudo_cl import plot_pseudo_cl_spectrum + +_unbuffer_streams() +p = snakemake.params + +spectra = {} +for i, version in enumerate(p["versions"]): + part = sacc_io.load(snakemake.input["pseudo_cl"][i]) + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(part, (0, 0)) + with fits.open(snakemake.input["pseudo_cl_cov"][i]) as hdul: + covs = { + name: np.asarray(hdul[f"COVAR_{name}_{name}"].data, float) + for name in ("EE", "EB", "BB") + } + for name, cl in (("EE", ee), ("EB", eb), ("BB", bb)): + spectra.setdefault(name, {})[version] = { + "ell": ell, + "cl": cl, + "cov": covs[name], + "style": {"marker": p["markers"][i], "colour": p["colours"][i]}, + } + +for name, datasets in spectra.items(): + plot_pseudo_cl_spectrum(datasets, name, snakemake.output[f"figure_{name.lower()}"]) + +verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py deleted file mode 100644 index 7dfebc54..00000000 --- a/workflow/scripts/cv_pseudo_cl.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Rule cv_pseudo_cl: harmonic-space pseudo-Cl B-mode spectra. - -plot_pseudo_cl triggers calculate_pseudo_cl, which writes pseudo_cl_{version}.fits -for every version (the BB spectrum cv_summarize_bmodes reads) and the cell_ee.png -figure. The per-version FITS files are the declared outputs. -""" - -from cv_runner import _unbuffer_streams, make_cv, verify_outputs - -_unbuffer_streams() -cv = make_cv(snakemake) -cv.plot_pseudo_cl() -verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index bcc46d84..1edaea93 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -1,24 +1,113 @@ """Rule cv_pure_eb: pure E/B-mode decomposition for one version. -Compute + plot rule (per version). plot_pure_eb calls calculate_pure_eb, which -runs two TreeCorr correlations (reporting + integration binning); the reporting -binning reuses the cv_2pcf data vector via calculate_2pcf's skip-if-exists -path. Writes the {version}_eb_..._data.npz data product (declared output) plus -companion figures, and the per-version E/B PTEs that cv_summarize_bmodes -collects. +A consumer of the two ξ± parts plus one covariance file — nothing here touches +a catalogue. The modes come from the reporting and integration parts through +the pipeline kernel; the covariance is Monte Carlo through that same kernel, +drawn from the CosmoCov integration-grid ξ± covariance around a theory mean, so +it depends on the covariance model and the grids rather than on the measured +vector. A jackknife of the transformed modes would need per-patch realisations, +which are never persisted. """ -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +import numpy as np +from cs_util.cosmo import get_cosmo +from cv_runner import _unbuffer_streams, verify_outputs + +from sp_validation import sacc_io +from sp_validation.b_modes import ( + calculate_eb_statistics, + log_bin_edges, + plot_eb_covariance_matrix, + plot_integration_vs_reporting, + plot_pte_2d_heatmaps, + plot_pure_eb_correlations, + pure_eb_covariance_mc, + pure_eb_from_xi, + save_pure_eb_results, +) +from sp_validation.cosmo_val.sacc_writers import pure_eb_to_sacc _unbuffer_streams() -cv = make_cv(snakemake) p = snakemake.params -cv.plot_pure_eb( - versions=[p["version"]], - min_sep_int=p["min_sep_int"], - max_sep_int=p["max_sep_int"], - nbins_int=p["nbins_int"], - fiducial_xip_scale_cut=tuple(p["fiducial_scale_cut"]), - fiducial_xim_scale_cut=tuple(p["fiducial_scale_cut"]), +version = p["version"] +fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) + +reporting = sacc_io.load(snakemake.input["xi_reporting"]) +integration = sacc_io.load(snakemake.input["xi_integration"]) +theta, xip, xim = sacc_io.get_xi(reporting, (0, 0), grid="reporting") +theta_int, xip_int, xim_int = sacc_io.get_xi(integration, (0, 0), grid="integration") +left_edges, right_edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) + +# The reporting grid must sit strictly inside the integration grid: a reporting +# point on the boundary has no interior support and comes back NaN. +modes = pure_eb_from_xi( + theta, xip, xim, theta_int, xip_int, xim_int, left_edges[0], right_edges[-1] +) + +z, nz = sacc_io.get_nz(reporting, 0) +cov, eb_samples = pure_eb_covariance_mc( + theta=theta, + left_edges=left_edges, + right_edges=right_edges, + theta_int=theta_int, + cov_int=np.loadtxt(snakemake.input["cov_integration"]), + z=z, + nz=nz, + cosmo=get_cosmo(**p["cosmo_params"]), + n_samples=p["n_samples"], +) + +variances = reporting.covariance.dense.diagonal() +results = { + "theta": theta, + "left_edges": left_edges, + "right_edges": right_edges, + "xip": xip, + "xim": xim, + "var_xip": variances[: len(theta)], + "var_xim": variances[len(theta) :], + "theta_int": theta_int, + "xip_int": xip_int, + "xim_int": xim_int, + "n_eff": p["n_samples"], + "cov": cov, + "eb_samples": eb_samples, + **modes, +} +results = calculate_eb_statistics(results) + +plot_integration_vs_reporting( + results, snakemake.output["figure_integration_vs_reporting"], version +) +plot_pure_eb_correlations( + results, + snakemake.output["figure_xis"], + version, + fiducial_xip_scale_cut=fiducial_scale_cut, + fiducial_xim_scale_cut=fiducial_scale_cut, +) +plot_pte_2d_heatmaps( + results, + version, + snakemake.output["figure_ptes"], + fiducial_xip_scale_cut=fiducial_scale_cut, + fiducial_xim_scale_cut=fiducial_scale_cut, +) +plot_eb_covariance_matrix( + cov, "semi-analytic", snakemake.output["figure_covariance"], version ) + +save_pure_eb_results(results, snakemake.output["npz"]) + +# The part inherits the ξ± part's provenance; `type` is re-stamped on save. +metadata = {k: v for k, v in reporting.metadata.items() if k != "type"} +s = pure_eb_to_sacc( + {0: (z, nz)}, + metadata, + theta, + {key: results[key] for key in sacc_io.PURE_KEYS}, + covariance=cov, +) +sacc_io.save(s, snakemake.output["sacc"], type="data") + verify_outputs(snakemake) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 85f0fa1d..9200bfaa 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -1,55 +1,63 @@ """Rule cv_summarize_bmodes: collect B-mode PTEs across all statistics. -The terminal diagnostic. summarize_bmodes reads the in-memory -_pure_eb_results / _cosebis_results / _pseudo_cls dicts, which are populated by -plot_pure_eb / plot_cosebis / plot_pseudo_cl. The per-version E/B and COSEBIs -npz products and the pseudo-Cl FITS are declared as inputs (so the DAG forces -those rules first), but the summary still needs the live result objects (it -reads each version's TreeCorr `gg`, which the npz cannot hold). So this rule -re-runs the three B-mode methods in-process: they reload the existing 2pcf / -data-vector files via their skip-if-exists paths and recompute only the cheap -PTE statistics, exactly as the original linear driver did on its shared cv. - -Writes the summary table to bmode_summary.txt (declared output) — the original -driver only printed it. +The terminal diagnostic, and a reader of what the three B-mode rules already +wrote: the pure-E/B PTE matrices and the COSEBIs B-mode PTE from their .npz +products, and the pseudo-Cℓ BB spectrum from its SACC part against the NaMaster +covariance. Nothing is recomputed and no catalogue is touched, so the summary +cannot disagree with the products it summarises. """ import json -from cv_runner import _unbuffer_streams, make_cv, verify_outputs +import numpy as np +from cv_runner import _unbuffer_streams, verify_outputs + +from sp_validation import sacc_io +from sp_validation.b_modes import _get_pte_from_scale_cut, log_bin_edges +from sp_validation.cosmo_val.core import print_bmode_summary +from sp_validation.statistics import chi2_and_pte _unbuffer_streams() -cv = make_cv(snakemake) p = snakemake.params fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) +edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) + +summary = {} +cov_methods = set() + +for i, version in enumerate(p["versions"]): + row = {} + + pure_eb = np.load(snakemake.input["pure_eb"][i]) + for stat in ("xip_B", "xim_B", "combined"): + try: + row[stat] = _get_pte_from_scale_cut( + pure_eb[f"pte_matrices_{stat}"], edges, fiducial_scale_cut + ) + except (KeyError, RuntimeError): + pass + cov_methods.add(f"pure-E/B: semi-analytic ({int(pure_eb['n_eff'])} draws)") + + # The COSEBIs .npz is written at the fiducial cut, so its PTE is the one + # this table wants. + cosebis = np.load(snakemake.input["cosebis"][i]) + row["COSEBIS"] = float(cosebis["pte_B"]) + cov_methods.add("COSEBIs: propagated from the ξ± covariance") + + if p["include_pseudo_cl"]: + from astropy.io import fits + + part = sacc_io.load(snakemake.input["pseudo_cl"][i]) + _ell, _ee, bb, _eb, _window = sacc_io.get_pseudo_cl(part, (0, 0)) + with fits.open(snakemake.input["pseudo_cl_cov"][i]) as hdul: + cov_bb = np.asarray(hdul["COVAR_BB_BB"].data, float) + _chi2, _red, row["C_l_BB"] = chi2_and_pte(bb, cov_bb) + cov_methods.add("pseudo-Cℓ: Gaussian (NaMaster)") + + summary[version] = row + +print_bmode_summary(summary, fiducial_scale_cut, cov_methods) -# Repopulate the in-memory B-mode result dicts from existing data products. -cv.plot_pure_eb( - min_sep_int=p["pure_eb_min_sep_int"], - max_sep_int=p["pure_eb_max_sep_int"], - nbins_int=p["pure_eb_nbins_int"], - fiducial_xip_scale_cut=fiducial_scale_cut, - fiducial_xim_scale_cut=fiducial_scale_cut, -) -for version in cv.versions: - cv.plot_cosebis( - version=version, - min_sep_int=p["cosebis_min_sep_int"], - max_sep_int=p["cosebis_max_sep_int"], - nbins_int=p["cosebis_nbins_int"], - npatch=p["cosebis_npatch"], - nmodes=p["cosebis_nmodes"], - scale_cuts=[tuple(sc) for sc in p["cosebis_scale_cuts"]], - fiducial_scale_cut=fiducial_scale_cut, - ) -if p.get("include_pseudo_cl", False): - cv.plot_pseudo_cl() - -summary = cv.summarize_bmodes(fiducial_scale_cut=fiducial_scale_cut) - -# summarize_bmodes prints its table and returns {version: {stat: pte}}. Persist -# the returned dict (the table itself is reproducible from it) so downstream -# tooling and the all-rule have a real, machine-readable artifact to depend on. with open(snakemake.output["summary_json"], "w") as f: json.dump(summary, f, indent=2, default=str) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a5c19a56..e25f6ed9 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -1,15 +1,11 @@ """Generate pseudo-Cls (data vector only, no covariance). Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` -object supplies the parameters and the native product is renamed to the tagged -output filename the rule declares; as a standalone CLI (argparse) the same -compute runs from explicit flags and the primitive's native -``pseudo_cl_{ver}.fits`` is left in place under ``--out`` (no rename — each -lc/ASTRA recipe gets its own output directory, so the untagged native name is -unambiguous and the primitives' skip-if-exists never collides across nbins -runs). The CLI form is what the lightcone/ASTRA recipe calls, so the -measurement is driven directly (no nested Snakemake) with lc handling -orchestration: +object supplies the parameters; as a standalone CLI (argparse) the same compute +runs from explicit flags. Either way the part is born at its final path, as +SACC (EE/BB/EB with a shared bandpower window). The CLI form is what the +lightcone/ASTRA recipe calls, driving the measurement directly (no nested +Snakemake) with lc handling orchestration: python generate_pseudo_cl.py \ --ver SP_v1.4.6.3_leak_corr \ @@ -28,14 +24,13 @@ import json import os -from astropy.io import fits - +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation def generate_pseudo_cl( version: str, - output_dir: str, + out_path: str, cat_config: str, nside: int = 1024, npatch: int = 1, @@ -45,16 +40,15 @@ def generate_pseudo_cl( nbins: int = None, power: float = 0.5, ): - """Generate a pseudo-Cl data vector into ``output_dir``. + """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. Parameters ---------- version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") - output_dir : str - Directory the pseudo-Cl FITS file is written into. The primitive writes - its native ``pseudo_cl_{version}.fits`` here; callers that need a tagged - filename rename it themselves (see ``_from_snakemake``). + out_path : str + Exact destination the SACC part is born at — its final (possibly tagged) + name. Skip-if-exists keys on it, so no two rules share a basename. cat_config : str Path to catalog configuration YAML nside : int @@ -76,8 +70,9 @@ def generate_pseudo_cl( Returns ------- str - Path to the primitive's native ``pseudo_cl_{version}.fits`` product. + ``out_path`` (the SACC part written). """ + output_dir = os.path.dirname(out_path) os.makedirs(output_dir, exist_ok=True) blind_str = f" blind={blind}" if blind else "" @@ -135,26 +130,23 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) - # Calculate pseudo-Cls only (no covariance) - cv.calculate_pseudo_cl() + # Pseudo-Cls only (no covariance), born directly at the final out_path. + cv.calculate_pseudo_cl(out_path=out_path) - # Report on the native product (renamed by the Snakemake caller, if any) - src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.fits") - if os.path.exists(src_cl): - with fits.open(src_cl) as hdul: - data = hdul["PSEUDO_CELL"].data - n_ell = len(data["ELL"]) - print(f"Generated pseudo-Cl with {n_ell} ell bins") - print(f"ell range: [{data['ELL'].min():.1f}, {data['ELL'].max():.1f}]") - return src_cl + if os.path.exists(out_path): + # Readback of the part just written — a legitimate pre-blind consumer. + s = sacc_io.load(out_path, allow_unblinded=True) + ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] + print(f"Generated pseudo-Cl with {len(ell)} ell bins") + print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") + return out_path def _from_snakemake(smk): p = smk.params - output_cl = smk.output.pseudo_cl - src_cl = generate_pseudo_cl( + generate_pseudo_cl( version=p["version"], - output_dir=os.path.dirname(output_cl), + out_path=smk.output.pseudo_cl, cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), @@ -164,10 +156,6 @@ def _from_snakemake(smk): nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), ) - # Snakemake declares a tagged output filename; rename the native product to it. - if os.path.exists(src_cl) and src_cl != output_cl: - os.rename(src_cl, output_cl) - print(f"Saved to: {output_cl}") def _from_cli(argv=None): @@ -220,9 +208,12 @@ def _from_cli(argv=None): with open(a.cosmo_json) as f: cosmo_params = json.load(f) + # lc/ASTRA path: --out is a per-recipe directory, so the untagged name is + # unambiguous there. + out_path = os.path.join(a.out, f"pseudo_cl_{a.ver}.sacc") generate_pseudo_cl( version=a.ver, - output_dir=a.out, + out_path=out_path, cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 86dffe27..1f69ee6d 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -12,15 +12,26 @@ --cat-config /path/to/cosmo_val/cat_config.yaml \ --out -The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump plus ξ+/ξ- FITS files into -``output_dir``. ``output_dir`` is passed explicitly (rather than via the -``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. +The measurement is binning-agnostic: the reporting and the fine integration +grids are the same compute with different ``--min-sep/--max-sep/--nbins``. +``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw +byproduct); the ξ± data product is born as SACC here, a *part* named by its +binning and tagged with its ``--grid``. The part carries the covariance its +grid configures (``--cov``): the dense jackknife estimate from the patches, the +TreeCorr ``varxip``/``varxim`` diagonal, or none. + +``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) +so lc can point each run at its own ``{output}`` tree. """ import argparse +import os + +import numpy as np +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc def run_2pcf( @@ -31,30 +42,68 @@ def run_2pcf( npatch, cat_config, output_dir, - save_fits=True, + sacc_out=None, + grid="reporting", + cov="none", ): - """Measure ξ±(θ) for ``ver`` and write it under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides - ``cat_config['paths']['output']`` so products land where lc expects. + ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc + expects. ``sacc_out`` is the exact destination for the SACC part (the + Snakemake-declared output); it defaults to a binning-derived name under + the resolved output directory for the CLI path. + + Returns + ------- + treecorr.GGCorrelation + The measured correlation object (also the source of the SACC part). """ cv = CosmologyValidation( versions=[ver], catalog_config=cat_config, output_dir=output_dir, + # so the SACC provenance metadata stamps the npatch actually measured + npatch=npatch, ) - return cv.calculate_2pcf( + gg = cv.calculate_2pcf( ver=ver, npatch=npatch, - save_fits=save_fits, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) + if cov == "jackknife" and int(npatch) < 2: + raise ValueError(f"cov='jackknife' needs patches; got npatch={npatch}") + + # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. + s = xi_to_sacc( + cv.sacc_nz(ver), + cv.sacc_metadata(ver), + gg.meanr, + gg.xip, + gg.xim, + grid=grid, + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + covariance=gg.cov if cov == "jackknife" else None, + variances=( + np.concatenate([gg.varxip, gg.varxim]) if cov == "diagonal" else None + ), + ) + out_path = sacc_out or os.path.join( + output_dir or cv.cc["paths"]["output"], + f"{ver}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc", + ) + sacc_io.save(s, out_path, type="data") + print(f"Wrote {grid} ξ± SACC part: {out_path}") + return gg + def _from_snakemake(smk): p = smk.params @@ -66,7 +115,11 @@ def _from_snakemake(smk): npatch=int(p["npatch"]), cat_config=p["cat_config"], output_dir=p["output_dir"], - save_fits=True, + grid=p.get("grid", "reporting"), + cov=p.get("cov", "none"), + # The SACC part goes exactly where the rule declares it; the .txt + # byproduct still lands under the resolved output dir. + sacc_out=smk.output["sacc"], ) @@ -93,7 +146,15 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument("--no-fits", action="store_true", help="Skip ξ+/ξ- FITS export") + ap.add_argument( + "--grid", default="reporting", help="SACC grid tag for the measured points" + ) + ap.add_argument( + "--cov", + default="none", + choices=["jackknife", "diagonal", "none"], + help="Covariance the part carries", + ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -103,7 +164,8 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - save_fits=not a.no_fits, + grid=a.grid, + cov=a.cov, ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index bc11bfc0..4983cffb 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -47,9 +47,10 @@ cv.calculate_rho_tau_stats() -# Confirm CosmologyValidation produced the requested outputs +# Confirm CosmologyValidation produced the requested outputs: the rho/tau FITS +# and the born-as-SACC rho_tau part. outputs = snakemake.output # type: ignore -for label in ("rho_stats", "tau_stats"): +for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) if not target.exists(): raise FileNotFoundError( From 71972c918a65d8aa83e6f92086d94d65e939f3f0 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Sat, 26 Sep 2026 00:49:48 +0200 Subject: [PATCH 30/83] fix(renovate): automerge pinned action digests (#353) Co-authored-by: Claude Opus 5.5 --- renovate.json | 1 + 1 file changed, 1 insertion(+) diff --git a/renovate.json b/renovate.json index 6a25bd4d..f0d14be6 100644 --- a/renovate.json +++ b/renovate.json @@ -30,6 +30,7 @@ "minor", "patch", "pin", + "pinDigest", "digest" ], "automerge": true From fca3c04b9a7f9dd253505e35f2187a3b14c2a7a1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 02:52:50 +0200 Subject: [PATCH 31/83] uv.lock: cs_util develop@a52280f, which restores FootprintPlotter.plot_area cv_footprints raised AttributeError: the locked cs_util (bedcdbf5) calls plot_area from plot_region but lacks it; cs_util f17ea0e restores it. The only other cs_util changes since are CI. Relocking also brings the lock's recorded sacc specifier in line with pyproject (>=2.4,<3). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index eb69507c..1460cbc9 100644 --- a/uv.lock +++ b/uv.lock @@ -691,7 +691,7 @@ wheels = [ [[package]] name = "cs-util" version = "0.2.2" -source = { git = "https://github.com/CosmoStat/cs_util.git?rev=develop#bedcdbf50ac57fc4d0b72a62b251373cd1807824" } +source = { git = "https://github.com/CosmoStat/cs_util.git?rev=develop#a52280fbb4bc1daa3f217a77954ba43c0860738b" } dependencies = [ { name = "astropy" }, { name = "camb" }, @@ -3951,7 +3951,7 @@ requires-dist = [ { name = "regions" }, { name = "reproject" }, { name = "ruff", marker = "extra == 'test'" }, - { name = "sacc", specifier = ">=0.12" }, + { name = "sacc", specifier = ">=2.4,<3" }, { name = "scipy", specifier = "<1.19" }, { name = "seaborn" }, { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, From 2ddeb6700c782fa853818d2e4afe2a01a6686a6b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 03:12:15 +0200 Subject: [PATCH 32/83] Dockerfile: sfmath in the image's TeX, for sans-serif usetex figures Checked in the #253 branch image: kpsewhich finds sfmath.sty, and a usetex figure with \usepackage[cm]{sfmath} renders. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 844f7e23..0437a929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,8 @@ RUN apt-get update -y --quiet --fix-missing && \ rm -rf /var/lib/apt/lists/* # TinyTeX pinned to a TeX Live year (frozen tlnet-final mirror); bump both once a year. +# The packages serve matplotlib's usetex figures; sfmath gives them sans-serif +# maths (`\usepackage[cm]{sfmath}`). ENV TEXLIVE_YEAR=2025 \ TINYTEX_VERSION=2026.02 \ TINYTEX_DIR=/opt \ @@ -46,6 +48,7 @@ RUN set -eux; \ amsmath \ amsfonts \ geometry \ + sfmath \ xcolor; \ tlmgr path add; \ latex --version >/dev/null; dvipng --version >/dev/null From f9a9a9dde1dd606856dedcca86ded89ee4ee60c0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:54:16 +0000 Subject: [PATCH 33/83] Update all non-major dependencies (#355) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8a0f226e..f4e87e9d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -86,7 +86,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then # can we push an autofix commit back to it with the workflow token. From 11b932236fc6661953764cb22c5fc65b357cac4a Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 05:50:16 +0200 Subject: [PATCH 34/83] papers/bmodes: drop the claims layer, the ecut and Moriond rules, and dead scripts (#358) --- papers/bmodes/Snakefile | 18 +- papers/bmodes/config/1d_plots.md | 49 --- papers/bmodes/config/2d_plots.md | 74 ---- .../bb_covariance_blind_independence.md | 34 -- papers/bmodes/config/bmodes_paper.md | 148 -------- papers/bmodes/config/cl.md | 34 -- papers/bmodes/config/cl_data_vector.md | 47 --- papers/bmodes/config/cl_version_comparison.md | 34 -- papers/bmodes/config/config.yaml | 19 +- .../config/config_space_pte_matrices.md | 55 --- papers/bmodes/config/cosebis.md | 50 --- papers/bmodes/config/cosebis_data_vector.md | 52 --- .../bmodes/config/cosebis_filter_overlay.md | 35 -- .../config/cosebis_version_comparison.md | 36 -- papers/bmodes/config/covariance.md | 88 ----- .../config/covariance_blind_consistency.md | 33 -- papers/bmodes/config/ecut_spec.md | 83 ----- .../harmonic_config_cosebis_comparison.md | 69 ---- .../config/harmonic_space_pte_matrices.md | 46 --- papers/bmodes/config/pure_eb.md | 46 --- papers/bmodes/config/pure_eb_covariance.md | 45 --- papers/bmodes/config/pure_eb_data_vector.md | 56 --- .../config/pure_eb_version_comparison.md | 38 --- papers/bmodes/config/xi_cosmology_paper.md | 110 ------ papers/bmodes/rules/ecut.smk | 218 ------------ .../bmodes/rules/{claims.smk => figures.smk} | 126 +------ papers/bmodes/rules/paper.smk | 70 ++++ papers/bmodes/rules/presentation.smk | 155 --------- papers/bmodes/rules/synthesis.smk | 141 -------- .../bb_covariance_blind_independence.py | 2 - papers/bmodes/scripts/cl_data_vector.py | 3 +- .../bmodes/scripts/cl_version_comparison.py | 3 +- .../scripts/compute_cosebis_pte_single.py | 2 +- .../scripts/config_space_pte_matrices.py | 7 +- papers/bmodes/scripts/container_env.sh | 45 --- .../scripts/cosebis_binning_comparison.py | 320 ------------------ papers/bmodes/scripts/cosebis_data_vector.py | 3 +- .../scripts/cosebis_version_comparison.py | 5 +- .../scripts/covariance_blind_consistency.py | 158 --------- .../explorations/compare_mask_effect.py | 158 --------- .../explorations/plot_mask_diagonal_ratio.py | 89 ----- .../scripts/filter_catalog_ellipticity.py | 86 ----- .../bmodes/scripts/generate_paper_macros.py | 53 +-- .../harmonic_config_cosebis_comparison.py | 13 +- .../scripts/harmonic_space_pte_matrices.py | 3 +- .../scripts/plot_cosebis_filter_overlay.py | 6 - .../scripts/plot_presentation_blind_nz.py | 39 --- .../bmodes/scripts/plot_pure_eb_covariance.py | 88 ----- papers/bmodes/scripts/pure_eb_covariance.py | 14 +- papers/bmodes/scripts/pure_eb_data_vector.py | 16 +- .../scripts/pure_eb_version_comparison.py | 14 +- papers/bmodes/scripts/run_cl_sweep.py | 113 ------- .../bmodes/scripts/run_cosebis_ptes_sweep.py | 99 ------ .../bmodes/scripts/run_pure_eb_ptes_sweep.sh | 48 --- .../scripts/run_pure_eb_semianalytic.sh | 80 ----- papers/bmodes/scripts/run_pure_eb_sweep.sh | 61 ---- papers/bmodes/scripts/run_xi_sweep.py | 85 ----- papers/bmodes/scripts/sweep_versions.py | 44 --- .../bmodes/scripts/test_bandpower_sampling.py | 205 ----------- papers/bmodes/scripts/update_survey_stats.py | 154 --------- .../scripts/validate_cosebis_filters.py | 265 --------------- .../bmodes/scripts/validate_cosebis_theory.py | 171 ---------- .../tests/test_bmodes_workflow_dry_run.py | 2 +- workflow/common.py | 3 +- workflow/scripts/plotting_utils.py | 2 +- 65 files changed, 127 insertions(+), 4341 deletions(-) delete mode 100644 papers/bmodes/config/1d_plots.md delete mode 100644 papers/bmodes/config/2d_plots.md delete mode 100644 papers/bmodes/config/bb_covariance_blind_independence.md delete mode 100644 papers/bmodes/config/bmodes_paper.md delete mode 100644 papers/bmodes/config/cl.md delete mode 100644 papers/bmodes/config/cl_data_vector.md delete mode 100644 papers/bmodes/config/cl_version_comparison.md delete mode 100644 papers/bmodes/config/config_space_pte_matrices.md delete mode 100644 papers/bmodes/config/cosebis.md delete mode 100644 papers/bmodes/config/cosebis_data_vector.md delete mode 100644 papers/bmodes/config/cosebis_filter_overlay.md delete mode 100644 papers/bmodes/config/cosebis_version_comparison.md delete mode 100644 papers/bmodes/config/covariance.md delete mode 100644 papers/bmodes/config/covariance_blind_consistency.md delete mode 100644 papers/bmodes/config/ecut_spec.md delete mode 100644 papers/bmodes/config/harmonic_config_cosebis_comparison.md delete mode 100644 papers/bmodes/config/harmonic_space_pte_matrices.md delete mode 100644 papers/bmodes/config/pure_eb.md delete mode 100644 papers/bmodes/config/pure_eb_covariance.md delete mode 100644 papers/bmodes/config/pure_eb_data_vector.md delete mode 100644 papers/bmodes/config/pure_eb_version_comparison.md delete mode 100644 papers/bmodes/config/xi_cosmology_paper.md delete mode 100644 papers/bmodes/rules/ecut.smk rename papers/bmodes/rules/{claims.smk => figures.smk} (85%) create mode 100644 papers/bmodes/rules/paper.smk delete mode 100644 papers/bmodes/rules/presentation.smk delete mode 100644 papers/bmodes/rules/synthesis.smk delete mode 100644 papers/bmodes/scripts/container_env.sh delete mode 100644 papers/bmodes/scripts/cosebis_binning_comparison.py delete mode 100644 papers/bmodes/scripts/covariance_blind_consistency.py delete mode 100644 papers/bmodes/scripts/explorations/compare_mask_effect.py delete mode 100644 papers/bmodes/scripts/explorations/plot_mask_diagonal_ratio.py delete mode 100644 papers/bmodes/scripts/filter_catalog_ellipticity.py delete mode 100644 papers/bmodes/scripts/plot_presentation_blind_nz.py delete mode 100644 papers/bmodes/scripts/plot_pure_eb_covariance.py delete mode 100644 papers/bmodes/scripts/run_cl_sweep.py delete mode 100644 papers/bmodes/scripts/run_cosebis_ptes_sweep.py delete mode 100644 papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh delete mode 100644 papers/bmodes/scripts/run_pure_eb_semianalytic.sh delete mode 100755 papers/bmodes/scripts/run_pure_eb_sweep.sh delete mode 100644 papers/bmodes/scripts/run_xi_sweep.py delete mode 100644 papers/bmodes/scripts/sweep_versions.py delete mode 100644 papers/bmodes/scripts/test_bandpower_sampling.py delete mode 100644 papers/bmodes/scripts/update_survey_stats.py delete mode 100644 papers/bmodes/scripts/validate_cosebis_filters.py delete mode 100644 papers/bmodes/scripts/validate_cosebis_theory.py diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index c9069c79..6af554bb 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -1,5 +1,5 @@ -# UNIONS B-modes paper — the epistemic layer, composed over the generic -# compute workflow at ../../workflow/. +# UNIONS B-modes paper (Paper III): figures, PTE tables and LaTeX macros, +# composed over the generic compute workflow at ../../workflow/. configfile: "config/config.yaml" configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" @@ -33,21 +33,17 @@ wildcard_constraints: **WILDCARD_CONSTRAINTS # Paper directories -CONFIG_DIR = "config" TAPESTRY_DIR = "results/tapestry" PAPER_FIGURES_DIR = "docs/unions_release/unions_bmodes/Figures" -# Compute rules (infrastructure — raw outputs, no evidence.json), -# imported as a module so future papers can compose the same workflow -# with their own config / overrides. +# Compute rules (raw measurements and covariances), imported as a module so +# other papers can compose the same workflow with their own config. module analysis: snakefile: os.path.join(WORKFLOW_DIR, "Snakefile") config: config use rule * from analysis -# Epistemic rules (evidence.json producers, order matters: claims → synthesis) -include: "rules/claims.smk" -include: "rules/ecut.smk" -include: "rules/synthesis.smk" -include: "rules/presentation.smk" +# Paper figures and their summary JSONs, then the macros/tables built from them +include: "rules/figures.smk" +include: "rules/paper.smk" diff --git a/papers/bmodes/config/1d_plots.md b/papers/bmodes/config/1d_plots.md deleted file mode 100644 index c5b45d84..00000000 --- a/papers/bmodes/config/1d_plots.md +++ /dev/null @@ -1,49 +0,0 @@ -# 1D Plotting Specification - -Shared styling for version comparisons, B-mode error bar plots, and diagnostic figures. - -## Purpose - -Consistent visual language across all 1D comparison plots. Fiducial version emphasized, comparison versions subdued. Clear error visualization with minimal clutter. - -## A&A Figure Dimensions - -Standard widths (inches): -- **Single-column**: 3.54 (88mm) -- **Double-column**: 7.24 (180mm) - -All figures use one of these two widths. Height varies by content. - -## Config References - -| Element | Config Key | -|---------|------------| -| Versions | `versions` | -| Fiducial | `fiducial.version` | -| Palette | `plotting.palette` | -| Fiducial alpha | `plotting.version_alpha.fiducial` | -| Comparison alpha | `plotting.version_alpha.comparison` | -| Marker style | `plotting.markers.style` | -| Line width | `plotting.markers.linewidth` | -| Cap size | `plotting.markers.capsize` | -| X-offsets | `plotting.x_offsets` | - -## Visual Design - -### Version Styling - -Fiducial version at full opacity, comparison versions subdued to create visual hierarchy. - -### Markers - -Unfilled circles with moderate line weight — clean, distinguishable at small sizes. Error caps sized for visibility without dominating. - -### Axes - -- **Zero line**: Black reference for null hypothesis -- **Grid**: Subtle guides on both axes -- **Mode highlight**: Background shading when relevant - -### Legend - -Bottom center, outside plot area. Single column for clarity. Framed for readability against varied backgrounds. diff --git a/papers/bmodes/config/2d_plots.md b/papers/bmodes/config/2d_plots.md deleted file mode 100644 index ebfba93b..00000000 --- a/papers/bmodes/config/2d_plots.md +++ /dev/null @@ -1,74 +0,0 @@ -# 2D Plotting Specification - -Shared styling for PTE heatmaps, covariance matrices, and 2D diagnostic plots. - -## Purpose - -Visualize parameter-space coverage (PTE grids) and correlation structure (covariance matrices). Emphasis on identifying outliers and structure. - -## A&A Figure Dimensions - -Standard widths (inches): -- **Single-column**: 3.54 (88mm) -- **Double-column**: 7.24 (180mm) - -Square heatmaps use single-column width (3.54 × 3.54). Larger matrices may use double-column. - -## Config References - -| Element | Config Key | -|---------|------------| -| PTE thresholds | `statistics.pte_healthy_range` | - -## PTE Heatmaps - -Grid search results showing PTE as function of two parameters. - -### Colormap - -`vlag` — diverging, emphasizes extremes (low/high PTE). - -### Contours - -Mark statistical significance at `statistics.pte_healthy_range` bounds — the 2σ equivalent thresholds. - -### Fiducial Marker - -Hatched rectangle highlighting the chosen fiducial configuration. - -## Axis Label Convention for Scale-Cut Heatmaps - -When plotting PTE matrices as function of scale cuts, axis labels must show the **included range boundary**: - -- **Lower-cut axis (x-axis)**: Show the **lower edge** of the included bin — the minimum scale actually included in the analysis -- **Upper-cut axis (y-axis)**: Show the **upper edge** of the included bin — the maximum scale actually included in the analysis - -This convention ensures the tick labels directly answer "what range is included in this analysis?" - -### Implementation - -**Config space** (angular scales): `theta_grid` is bin edges (21 values for 20 bins). -- x-axis (θ_min): use `theta_grid[i]` (lower edge) -- y-axis (θ_max): use `theta_grid[i+1]` (upper edge) - -**Harmonic space** (multipoles): `ell` from FITS is bin centers. Compute edges from binning config: -```python -def compute_ell_edges(lmin, lmax, n_bins, power=0.5): - start = np.power(lmin, power) - end = np.power(lmax, power) - return np.power(np.linspace(start, end, n_bins + 1), 1 / power) -``` -- x-axis (ℓ_min): use `ell_edges[i]` (lower edge) -- y-axis (ℓ_max): use `ell_edges[i+1]` (upper edge) - -## Covariance Matrices - -Visualize E/B mode covariance structure. - -### Colormap - -`icefire` — symmetric diverging for positive/negative correlations. - -### Structure - -Show block separation between E-modes, B-modes, and ambiguous modes. Highlight cross-correlations between mode types. diff --git a/papers/bmodes/config/bb_covariance_blind_independence.md b/papers/bmodes/config/bb_covariance_blind_independence.md deleted file mode 100644 index 33ec2cb9..00000000 --- a/papers/bmodes/config/bb_covariance_blind_independence.md +++ /dev/null @@ -1,34 +0,0 @@ -# BB Covariance Blind Independence - -Method: [Covariance](covariance.md), [Pure E/B](pure_eb.md), [COSEBIS](cosebis.md), [Cl](cl.md) - -## Claim - -BB covariances computed via **analytic propagation** are blind-independent, while BB covariances computed via **MC sampling** inherit noise that varies across blinds. - -## Evidence - -Diagonal ratios relative to blind A for both B-modes and E-modes: -- Pure E/B: `cov(ξ+^B)`, `cov(ξ-^B)` vs `cov(ξ+^E)`, `cov(ξ-^E)` -- COSEBIS: `cov(B_n)` vs `cov(E_n)` -- Harmonic: `cov(C_ℓ^BB)` vs `cov(C_ℓ^EE)` - -**Metrics:** -1. Diagonal ratio plot — ratio of B/A and C/A diagonals, report min/max -2. Comparison of BB vs EE deviations across methods - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Version | `fiducial.version` | -| Scale range | `fiducial.min_sep` to `fiducial.max_sep` | -| Bins | `fiducial.nbins` | -| COSEBIS nmodes | `fiducial.nmodes` | -| COSEBIS θ range | `cosebis.theta_min` to `cosebis.theta_max` | -| n_ell_bins | `cl.n_ell_bins` | - -## Outputs - -- `evidence.json` — ratios, deviations, comparison statistics -- `figure.png` — multi-panel ratio plot comparing BB vs EE stability diff --git a/papers/bmodes/config/bmodes_paper.md b/papers/bmodes/config/bmodes_paper.md deleted file mode 100644 index 5623fe8a..00000000 --- a/papers/bmodes/config/bmodes_paper.md +++ /dev/null @@ -1,148 +0,0 @@ -# B-modes Paper - -The B-modes paper (Daley et al.) — B-mode validation for UNIONS cosmic shear. - -Depends on: [pure_eb_covariance](pure_eb_covariance.md), [pure_eb_data_vector](pure_eb_data_vector.md), [pure_eb_version_comparison](pure_eb_version_comparison.md), [cosebis_version_comparison](cosebis_version_comparison.md) (incl. `paper_stacked` output), [cosebis_data_vector](cosebis_data_vector.md), [cl_data_vector](cl_data_vector.md), [cl_version_comparison](cl_version_comparison.md), [config_space_pte_matrices](config_space_pte_matrices.md), [harmonic_space_pte_matrices](harmonic_space_pte_matrices.md), [bb_covariance_blind_independence](bb_covariance_blind_independence.md), [covariance_blind_consistency](covariance_blind_consistency.md) - -## Scope - -Three statistics, multiple catalog versions (from `config.versions`), one question: are B-modes consistent with noise? - -The config-space paper (Goh et al.) reports the answer for the fiducial catalog. This paper shows the work — methodology, version comparison, scale cut selection. - -## Figure Mapping - -### Main Text — Methods - -| Figure | Claim | File | Section | Label | -|--------|-------|------|---------|-------| -| Pure E/B decomposition | `pure_eb_data_vector` | `pure_eb_data_vector.png` | §2.3 | `fig:pure_eb_decomposition` | -| COSEBIS B-modes | `cosebis_data_vector` | `cosebis_data_vector.png` | §2.3 | `fig:cosebis_fiducial` | -| Harmonic fiducial | `cl_data_vector` | `cl_fiducial.png` | §2.3 | `fig:cl_fiducial` | -| Pure E/B covariance | `pure_eb_covariance` | `eb_covariance.png` | §2.4 | `fig:eb_covariance` | - -### Main Text — Results - -| Figure | Claim | File | Section | Label | -|--------|-------|------|---------|-------| -| Config-space PTE heatmaps | `config_space_pte_matrices` | `config_space_pte_fiducial.png` | §3 | `fig:pte_heatmaps` | -| Harmonic PTE heatmap | `harmonic_space_pte_matrices` | `cl_pte_heatmap.png` | §3 | `fig:pte_cl` | -| Pure E/B version comparison | `pure_eb_version_comparison` | `pure_eb_versions.png` | §3.1 | `fig:pure_eb_versions` | -| Harmonic version comparison | `cl_version_comparison` | `cl_versions.png` | §3.1 | `fig:cl_versions` | - -### Appendix - -| Figure | Claim | File | Section | Label | -|--------|-------|------|---------|-------| -| Config-space PTE (all versions) | `config_space_pte_matrices` | `config_space_pte_composite_appendix.png` | Appendix A | `fig:pte_appendix` | -| Harmonic PTE (all versions) | `harmonic_space_pte_matrices` | `cl_pte_composite_appendix.png` | Appendix A | `fig:pte_cl_appendix` | - -Figures copy from `results/tapestry/{claim}/` to `docs/unions_release/unions_bmodes/Figures/`. - -## Pure E/B Covariance - -Evidence: `results/tapestry/pure_eb_covariance/evidence.json` - -### Block Structure - -Six blocks of 20 bins each, ordered: ξ+^E, ξ-^E, ξ+^B, ξ-^B, ξ+^amb, ξ-^amb. - -The evidence groups these into three 40×40 analysis blocks: -- `xi_E`: ξ+^E and ξ-^E combined -- `xi_B`: ξ+^B and ξ-^B combined -- `xi_amb`: ξ+^amb and ξ-^amb combined - -### Figure Context - -Appears in §2.4 after COSEBIS covariance discussion: - -> Figure~\ref{fig:eb_covariance} shows the pure E/B covariance structure. - -Caption references `covariance.n_samples` (currently 2000). - -## PTE Reporting - -### Format - -Two significant figures: "PTE of 0.55" - -Scientific notation below 0.01: "PTE of 2.6 × 10⁻⁴" - -### Scale Cuts - -En-dash, arcmin units: "6–85 arcmin" - -### Tables - -Bold failures (< 0.01). Values from evidence files, not hardcoded. - -Example structure (values populated from evidence): - -| Version | ξ+^B | ξ-^B | Joint | -|---------|------|------|-------| -| v1.4.X | `config_space_pte_matrices.evidence.versions[ver].xip_stats...` | ... | ... | - -Note: Per-version PTEs come from `config_space_pte_matrices`, not `pure_eb_data_vector`. - -### Blind Handling - -PTEs report the **minimum across blinds** (`pte_joint_min`) as the conservative estimate. This ensures reported values remain valid regardless of which blind is eventually unblinded. The fiducial blind `config["fiducial"]["blind"]` determines covariance matrix selection. - -## Macros and Tables - -Generated by `paper_macros` rule: - -| Output | Purpose | -|--------|---------| -| `claims_macros.tex` | LaTeX macro definitions (`\ebfiducialPte`, etc.) | -| `pte_table_results.tex` | PTE results table for main text | -| `pte_table_appendix.tex` | PTE table for appendix (all versions) | - -All outputs written to `docs/unions_release/unions_bmodes/`. - -### Pure E/B Covariance - -| Macro | Evidence Path | -|-------|---------------| -| `\ebcovCondE` | `pure_eb_covariance` → `block_analysis.xi_E.condition_number` | -| `\ebcovCondB` | `pure_eb_covariance` → `block_analysis.xi_B.condition_number` | -| `\ebcovCondAmb` | `pure_eb_covariance` → `block_analysis.xi_amb.condition_number` | -| `\ebcovCondFull` | `pure_eb_covariance` → `condition_number` | -| `\ebcovNbins` | `pure_eb_covariance` → `n_bins` | - -### Pure E/B Decomposition - -| Macro | Evidence Path | -|-------|---------------| -| `\ebfiducialPte` | `pure_eb_data_vector` → `fiducial.pte_joint_min` | -| `\ebfullPte` | `pure_eb_data_vector` → `full.pte_joint_min` | -| `\ebthetaXipMin`, `\ebthetaXipMax` | `pure_eb_data_vector` → `fiducial.scale_cut_xip` | -| `\ebthetaXimMin`, `\ebthetaXimMax` | `pure_eb_data_vector` → `fiducial.scale_cut_xim` | - -### COSEBIS - -| Macro | Evidence Path | -|-------|---------------| -| `\cosebisfiducialPte` | `cosebis_version_comparison` → `fiducial.versions.{fiducial_version}.pte_6_min` | -| `\cosebisfullPte` | `cosebis_version_comparison` → `full.versions.{fiducial_version}.pte_6_min` | -| `\cosebisthetaMin`, `\cosebisthetaMax` | `cosebis_version_comparison` → `fiducial.scale_cut_arcmin` | - -### Covariance Consistency - -| Macro | Evidence Path | -|-------|---------------| -| `\covXipMaxDev` | `covariance_blind_consistency` → max deviation across blinds | -| `\covXimMaxDev` | `covariance_blind_consistency` → max deviation across blinds | -| `\ebJointPteDelta` | `pure_eb_data_vector` → joint PTE variation (max − min) | - -Never hardcode values in text. Use macros. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Fiducial version | `fiducial.version` | -| MC samples | `covariance.n_samples` | -| Reporting bins | `fiducial.nbins` | -| ξ+ scale cut | `fiducial.fiducial_xip_scale_cut` | -| ξ- scale cut | `fiducial.fiducial_xim_scale_cut` | diff --git a/papers/bmodes/config/cl.md b/papers/bmodes/config/cl.md deleted file mode 100644 index ad0b7d70..00000000 --- a/papers/bmodes/config/cl.md +++ /dev/null @@ -1,34 +0,0 @@ -# Harmonic-Space Power Spectra - -Pseudo-Cl estimation of cosmic shear E/B-mode power spectra using NaMaster. - -## Method - -Angular power spectra C_ell are the harmonic-space analog of configuration-space correlation functions. For cosmic shear: - -- **C_ell^EE**: E-mode auto-power spectrum (cosmological signal) -- **C_ell^BB**: B-mode auto-power spectrum (null test for systematics) -- **C_ell^EB**: E-B cross-power spectrum (should be zero) - -Pseudo-Cl estimation accounts for: -1. Survey mask geometry via mode-coupling matrices -2. Bandpower binning (sqrt(ell)-linear spacing) -3. Noise bias subtraction - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| ell bins | `cl.n_ell_bins` | -| Fiducial version | `fiducial.version` | - -## Data Source - -Pseudo-Cl files generated by workflow rules using NaMaster: -- `pseudo_cl_{version}_blind={blind}_powspace_nbins={nbins}.sacc` — Power spectrum estimates -- `pseudo_cl_cov_{version}_blind={blind}_powspace_nbins={nbins}.fits` — Bandpower covariance matrix - -Location: `{COSMO_VAL_OUTPUT}/` (defined in Snakefile, typically `/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/`) - -Default parameters: `blind=A` (from `fiducial.blind`), `nbins=32` (from `cl.n_ell_bins`). - diff --git a/papers/bmodes/config/cl_data_vector.md b/papers/bmodes/config/cl_data_vector.md deleted file mode 100644 index 4863f064..00000000 --- a/papers/bmodes/config/cl_data_vector.md +++ /dev/null @@ -1,47 +0,0 @@ -# Harmonic-Space Fiducial - -Depends: [Harmonic-Space Power Spectra](cl.md), [2D Plots](2d_plots.md) -Method: [Harmonic-Space Power Spectra](cl.md) - -## Claim - -The fiducial catalog shows B-mode power spectra consistent with zero across all ell bins, validating the absence of significant systematic contamination in harmonic space. PTEs are reported both for the full ell range and with scale cuts (ell_min=300, ell_max=1600) from the harmonic space paper. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Fiducial version | `fiducial.version` | -| ell bins | `cl.n_ell_bins` | -| Scale cuts | `cl.fiducial_ell_min`, `cl.fiducial_ell_max` (from the harmonic paper, Guerrini et al.) | - -## Evidence - -Power spectra and PTEs for BB and EB components, with and without scale cuts: - -| Metric | Description | -|--------|-------------| -| `pte_bb_full` | B-mode PTE, full ell range | -| `pte_eb_full` | E-B cross PTE, full ell range | -| `pte_bb_cut` | B-mode PTE with scale cuts | -| `pte_eb_cut` | E-B cross PTE with scale cuts | -| `version` | Fiducial catalog version | - -## Outputs - -Produces 9 figures: 1 paper figure + 4 per-version leak-corrected + 4 per-version uncorrected. - -**Paper figure (leak-corrected, fiducial version):** -- `figure.png` — Two-panel figure (BB top, EB bottom), no title -- BB: filled circles, EB: unfilled squares -- Data normalized by errors (C_ell / sigma) -- Scale cuts marked with shaded excluded regions -- sqrt(ell) x-axis scaling matches bandpower binning - -**Per-version figures (leak-corrected, with title):** -- `figure_v{X.Y.Z}.png` — One per leak-corrected version, with version title - -**Per-version figures (uncorrected, with title):** -- `figure_v{X.Y.Z}_uncorrected.png` — One per uncorrected version, with version title -- Labeled "(uncorrected)" in legend -- For validation/comparison purposes, not included in paper diff --git a/papers/bmodes/config/cl_version_comparison.md b/papers/bmodes/config/cl_version_comparison.md deleted file mode 100644 index 07c4e0c8..00000000 --- a/papers/bmodes/config/cl_version_comparison.md +++ /dev/null @@ -1,34 +0,0 @@ -# Harmonic-Space Version Comparison - -Depends: [Harmonic-Space Power Spectra](cl.md), [2D Plots](2d_plots.md) -Method: [Harmonic-Space Power Spectra](cl.md) -Plotting: [2D Plots](2d_plots.md) - -## Claim - -B-mode power spectra C_ell^BB are consistent with zero for all leak-corrected versions. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Versions | `versions` | -| ell bins | `cl.n_ell_bins` | -| PTE healthy range | `statistics.pte_healthy_range` | - -## Evidence - -Per-version B-mode PTEs: - -| Metric | Description | -|--------|-------------| -| `{version}.pte_bb` | B-mode power spectrum PTE | -| `{version}.pte_eb` | E-B cross PTE | -| `{version}.chi2_bb` | B-mode chi-squared | -| `{version}.dof` | Degrees of freedom | - -## Outputs - -- `figure.png` — Two-panel plot (BB top, EB bottom) with all versions overlaid -- Version comparison uses distinct colors/markers -- sqrt(ell) x-axis scaling diff --git a/papers/bmodes/config/config.yaml b/papers/bmodes/config/config.yaml index 47b385bd..ea51a89f 100644 --- a/papers/bmodes/config/config.yaml +++ b/papers/bmodes/config/config.yaml @@ -1,12 +1,9 @@ versions: [ "SP_v1.4.5", "SP_v1.4.5_leak_corr", "SP_v1.4.6.3", "SP_v1.4.6.3_leak_corr", - "SP_v1.4.6_ecut07", "SP_v1.4.6_ecut07_leak_corr", "SP_v1.4.8", "SP_v1.4.8_leak_corr", - "SP_v1.4.11.3", "SP_v1.4.11.3_leak_corr", - "SP_v1.4.11.3_ecut07", "SP_v1.4.11.3_ecut07_leak_corr" + "SP_v1.4.11.3", "SP_v1.4.11.3_leak_corr" ] -# IV weight variants removed — one-off test (see fiber v1-4-11-2-iv-weights-harmonic-b-2a2ed88b) ellipticity_suffix: "_leak_corrected" # grid search parameters @@ -153,17 +150,3 @@ glass_mocks: # External tools tools: cosmocov_executable: "/n23data1/n06data/lgoh/scratch/UNIONS/CosmoCov/covs/cov" - -# Ellipticity cut investigation -ecut: - versions_for_comparison: - - SP_v1.4.6.3_leak_corr - - SP_v1.4.6_ecut07_leak_corr - - SP_v1.4.11.3_leak_corr - - SP_v1.4.11.3_ecut07_leak_corr - version_labels: - SP_v1.4.6.3_leak_corr: "v1.4.6.3" - SP_v1.4.6_ecut07_leak_corr: "v1.4.6 $|e|<0.7$" - SP_v1.4.11.3_leak_corr: "v1.4.11.3" - SP_v1.4.11.3_ecut07_leak_corr: "v1.4.11.3 $|e|<0.7$" - fiducial_for_comparison: "SP_v1.4.6.3_leak_corr" diff --git a/papers/bmodes/config/config_space_pte_matrices.md b/papers/bmodes/config/config_space_pte_matrices.md deleted file mode 100644 index 40073add..00000000 --- a/papers/bmodes/config/config_space_pte_matrices.md +++ /dev/null @@ -1,55 +0,0 @@ -# Configuration-Space PTE Matrices - -Depends: [Pure E/B](pure_eb.md), [COSEBIS](cosebis.md), [2D Plots](2d_plots.md) -Method: [Pure E/B](pure_eb.md), [COSEBIS](cosebis.md) -Plotting: [2D Plots](2d_plots.md) - -## Claim - -Fiducial angular scale cuts are justified by PTE heatmaps across all (theta_min, theta_max) combinations. The main text presents a 1x3 composite for the fiducial version showing xi+^B, xi-^B, and COSEBIS B_n. The appendix presents an Nx3 composite showing all catalog versions (from `config.versions`) with each of the three B-mode statistics. Rows represent versions (labeled via `config.plotting.version_labels`), columns represent statistics. - -Scale cuts from `fiducial.fiducial_xip_scale_cut` and `fiducial.fiducial_xim_scale_cut`. COSEBIS uses the same unified range. - -## Blind Handling - -Uses fiducial blind from `config["fiducial"]["blind"]`. Data vectors (ξ+^B, ξ-^B, COSEBIS B_n) are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Versions | `versions` | -| Fiducial version | `fiducial.version` | -| xi+ scale cut | `fiducial.fiducial_xip_scale_cut` | -| xi- scale cut | `fiducial.fiducial_xim_scale_cut` | -| COSEBIS scale cut | `fiducial.fiducial_min_scale`, `fiducial.fiducial_max_scale` | - -## Evidence - -Per-version statistics for each statistic: - -| Metric | Description | -|--------|-------------| -| `{version}.role` | "fiducial" or "appendix" | -| `{version}.xip_stats.pte_at_fiducial` | xi+^B PTE at fiducial scale cut | -| `{version}.xip_stats.pte_at_full_range` | xi+^B PTE at full theta range | -| `{version}.xim_stats.pte_at_fiducial` | xi-^B PTE at fiducial scale cut | -| `{version}.xim_stats.pte_at_full_range` | xi-^B PTE at full theta range | -| `{version}.cosebis_stats.pte_at_fiducial` | COSEBIS B_n PTE at fiducial | -| `{version}.cosebis_stats.pte_at_full_range` | COSEBIS B_n PTE at full theta range | - -## Outputs - -- `figure_fiducial.png` — 1x3 composite for fiducial version (main text) - - Single row: xi+^B, xi-^B, COSEBIS B_n - - Y-axis label on leftmost panel - - Single shared colorbar on right - -- `figure_appendix.png` — Nx3 composite for all versions (appendix) - - Rows: All versions from `config.versions` with labels from `config.plotting.version_labels` - - Columns: xi+^B, xi-^B, COSEBIS B_n - - Y-axis label on each row's leftmost panel - - Version labels on right side of each row - - Single shared colorbar on right - -Each panel uses a discrete PTE colormap (`make_pte_colormap` from `plotting_utils.py`) with solid blue below 0.05, solid red above 0.95, and a gradient between. No contour overlays. Fiducial scale cut marked with a plain black-edged rectangle (no hatching). diff --git a/papers/bmodes/config/cosebis.md b/papers/bmodes/config/cosebis.md deleted file mode 100644 index 25756887..00000000 --- a/papers/bmodes/config/cosebis.md +++ /dev/null @@ -1,50 +0,0 @@ -# COSEBIS Specification - -Complete Orthogonal Sets of E/B-Integrals — a model-independent B-mode null test. - -## Purpose - -COSEBIS compress 2PCF information into discrete modes with clean E/B separation. Unlike band powers, they're complete and orthogonal over a finite angular range. B-modes should be consistent with zero for a pure lensing signal. - -## Config References - -| Parameter | Config Key | Description | -|-----------|------------|-------------| -| n_modes | `fiducial.nmodes` | Number of COSEBIS modes to compute | -| θ_min | `cosebis.theta_min` | Minimum angular scale (arcmin) | -| θ_max | `cosebis.theta_max` | Maximum angular scale (arcmin) | -| mode_subsets | `cosebis.mode_subsets` | Mode counts for PTE evaluation | -| scale_cut | `fiducial.fiducial_min_scale` to `fiducial.fiducial_max_scale` | Fiducial scale range | -| pte_range | `statistics.pte_healthy_range` | Healthy PTE bounds | - -## Integration Grid - -High-resolution 2PCF for accurate mode integration: - -| Parameter | Config Key | -|-----------|------------| -| min_sep | `fiducial.min_sep_int` | -| max_sep | `fiducial.max_sep_int` | -| nbins | `fiducial.nbins_int` | - -## Versions - -Compare across catalog versions from `config.versions`: -- Fiducial: `fiducial.version` -- All versions tested for consistency - -## Blind Handling - -Uses fiducial blind from `config["fiducial"]["blind"]`. COSEBIS B_n data vectors are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. - -## Analysis Decisions - -- **No Hartlap correction**: Using theoretical (CosmoCov) covariance, not jackknife -- **Conservative scale cuts**: Require full bin containment within θ range -- **PTE thresholds**: Values outside healthy range flag potential systematics - -## Outputs - -- B-mode plot with fiducial scale cuts -- B-mode plot without scale cuts (full range) -- PTE values for each mode subset diff --git a/papers/bmodes/config/cosebis_data_vector.md b/papers/bmodes/config/cosebis_data_vector.md deleted file mode 100644 index 064ae63a..00000000 --- a/papers/bmodes/config/cosebis_data_vector.md +++ /dev/null @@ -1,52 +0,0 @@ -# COSEBIs Data Vector - -Depends: [COSEBIS](cosebis.md), [1D Plots](1d_plots.md) -Method: [COSEBIS](cosebis.md) -Plotting: [1D Plots](1d_plots.md) - -## Claim - -COSEBIS B-modes at fiducial version (`fiducial.version`) are consistent with zero across the full angular range and at fiducial scale cuts. - -## Blind Handling - -Uses fiducial blind from `config["fiducial"]["blind"]`. COSEBIS B_n data vectors are identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Fiducial version | `fiducial.version` | -| Fiducial scale cut | `fiducial.fiducial_min_scale` to `fiducial.fiducial_max_scale` | -| Number of modes | `fiducial.nmodes` | - -## Evidence - -This claim produces visualizations only. Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). - -| Metric | Description | -|--------|-------------| -| `fiducial_scale_cut` | Angular range for fiducial | -| `full_scale_cut` | Full angular range | -| `nmodes` | Number of COSEBIS modes | - -## Outputs - -Produces 9 figures: 1 paper figure + 4 per-version leak-corrected + 4 per-version uncorrected. - -**Paper figure (leak-corrected, fiducial version):** -- `figure.png` — Single-panel figure showing $B_n / \sigma_n$ for fiducial version, no title - - Both scale cuts (fiducial and full) overplotted with different colors - - Error bars are unity by construction (normalized) - - Paper figure for main text B-mode validation - -**Per-version figures (leak-corrected, with title):** -- `figure_v{X.Y.Z}.png` — One per leak-corrected version, with version title - -**Per-version figures (uncorrected, with title):** -- `figure_v{X.Y.Z}_uncorrected.png` — One per uncorrected version, with version title -- For validation/comparison purposes, not included in paper - -## Notes - -This is the paper-ready data vector figure. For multi-version comparison, see [COSEBIS Version Comparison](cosebis_version_comparison.md). diff --git a/papers/bmodes/config/cosebis_filter_overlay.md b/papers/bmodes/config/cosebis_filter_overlay.md deleted file mode 100644 index 9e263e3d..00000000 --- a/papers/bmodes/config/cosebis_filter_overlay.md +++ /dev/null @@ -1,35 +0,0 @@ -# COSEBIS Filter Overlay - -## Claim - -The COSEBIS harmonic-space filter functions W_n(ell) become increasingly oscillatory -for higher modes n, making them impossible to resolve with 32 coarse bandpower bins. - -## Method - -Overlay the continuous W_n(ell) filter functions (modes n = 1 through 6) on the -measured 32-bin BB bandpower spectrum from the fiducial catalog. Show both the full -[1', 250'] and fiducial [12', 83'] angular scale cuts. - -W_n(ell) computed via cosmo_numba's FFT-log Hankel transform on a dense ell grid -(2000 points). Each filter peak-normalized for visual comparison. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Full theta range | `cosebis.theta_min`, `cosebis.theta_max` | -| Fiducial theta range | `fiducial.fiducial_min_scale`, `fiducial.fiducial_max_scale` | -| Number of ell bins | `cl.n_ell_bins` | - -## Evidence - -| Metric | Description | -|--------|-------------| -| `nmodes_shown` | Number of COSEBIS modes displayed | -| `n_bandpower_bins` | Number of bandpower bins in data | -| `ell_range` | Effective ell range of the bandpowers | - -## Outputs - -- `figure.png` — Two-panel overlay: W_n filters + BB data for full and fiducial scale cuts diff --git a/papers/bmodes/config/cosebis_version_comparison.md b/papers/bmodes/config/cosebis_version_comparison.md deleted file mode 100644 index e02e3db8..00000000 --- a/papers/bmodes/config/cosebis_version_comparison.md +++ /dev/null @@ -1,36 +0,0 @@ -# COSEBIS Version Comparison - -Depends: [COSEBIS](cosebis.md), [1D Plots](1d_plots.md) -Method: [COSEBIS](cosebis.md) -Plotting: [1D Plots](1d_plots.md) - -## Claim - -B-mode COSEBIS magnitudes are small relative to measurement uncertainty across catalog versions. Visualization shows $B_n / \sigma_n$ (in units of standard deviation from zero) at fiducial and full angular ranges. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Versions | `versions` | -| Fiducial | `fiducial.version` | -| Scale cut | `fiducial.fiducial_min_scale` to `fiducial.fiducial_max_scale` | - -## Evidence - -This claim produces visualizations only. Statistical evidence (PTEs) is in [Config-Space PTE Matrices](config_space_pte_matrices.md). - -| Metric | Description | -|--------|-------------| -| `scale_cuts` | Angular ranges shown | -| `versions_plotted` | Catalog versions included | - -## Outputs - -Main figure shows leak_corr catalog versions from `config.versions` for catalog evolution comparison. - -- `figure_stacked.png` — Two-panel figure showing $B_n / \sigma_n$ (catalog evolution) - - Top: Full range (no scale cuts) - - Bottom: Fiducial scale cut - - Error bars are unity by construction (normalized) - - Legend labels from `config.version_labels` diff --git a/papers/bmodes/config/covariance.md b/papers/bmodes/config/covariance.md deleted file mode 100644 index 54e7ad42..00000000 --- a/papers/bmodes/config/covariance.md +++ /dev/null @@ -1,88 +0,0 @@ -# Covariance - -Semi-analytical covariance estimation using CosmoCov. - -## Purpose - -Provides theoretical covariance matrices for ξ± correlation functions. Avoids jackknife noise by propagating analytical Gaussian covariance through the analysis pipeline. - -## Method - -1. **CosmoCov** computes Gaussian-only covariance for integration binning -2. **Sampling** draws realizations from integration covariance -3. **Propagation** bins samples to reporting scale, propagates through E/B decomposition - -Gaussian-only approximation: non-Gaussian contributions computationally prohibitive at 1000 bins. - -## Config References - -| Parameter | Source | Description | -|-----------|--------|-------------| -| Samples | `config.yaml: covariance.n_samples` | MC samples for propagation (default 2000) | -| Use masked | `config.yaml: covariance.default_masked` | Whether to use masked covariance | -| Cosmology | `Snakefile: PLANCK18` | astropy Planck18 via cs_util.cosmo | -| Mask Cls | `covariance.smk: MASK_CLS_FILES` | Per-version mask power spectra | - -## Survey Properties - -Extracted from catalog config (`cat_config.yaml`) per version: -- Area (deg²) -- n_e (gal/arcmin²) -- σ_e (shape noise) - -## Binning - -| Scale | min | max | bins | -|-------|-----|-----|------| -| Integration | 0.5' | 500' | 1000 | -| Reporting | 1' | 250' | 20 | - -## File Naming - -``` -covariance_{version}_{blind}_{gaussian}_minsep={min}_maxsep={max}_nbins={n}{mask_suffix}_processed.txt -``` - -- `version`: SP_v1.4.6_leak_corr, etc. -- `blind`: A, B, or C -- `gaussian`: g (Gaussian-only) or ng (non-Gaussian) -- `mask_suffix`: empty or `_masked` - -## Blind Handling - -B-mode claims use the fiducial blind from `config["fiducial"]["blind"]`. Covariances are computed for the fiducial blind only. - -## Covariance Usage Policy - -Official results use specific covariance sources for consistency and correctness: - -| Quantity | Source | Gaussian | Binning | Notes | -|----------|--------|----------|---------|-------| -| Total ξ± errors | CosmoCov | ng | 20-bin (reporting) | Per-blind, masked | -| Pure E/B mode errors | MC propagation | g | 1000→20 bin | Conservative: underestimates uncertainty | -| COSEBIS errors | MC propagation | g | 1000→scales | Per scale cut | - -**Not used:** -- TreeCorr jackknife covariance — too noisy for official results -- NPZ `cov_xip_xim` field — deprecated (was jackknife) - -### Footprint Masks - -Two footprint masks at nside=4096, generated from the comprehensive catalog with -only spatially-structured cuts (no galaxy selection cuts): - -| Mask | Area | Used by | -|------|------|---------| -| Standard footprint | 2894 deg² | v1.4.5, v1.4.6, v1.4.11.3 (and ecut variants) | -| Star-halo footprint | 2517 deg² | v1.4.8 | - -Each version gets its own covariance from its own survey properties (A, n_e, sigma_e). -`resolve_covariance_version()` is the identity function — no cross-version covariance sharing. -`MASK_CLS_FILES` (covariance.smk) maps to two mask power spectrum files based on whether -the version is in `STARHALO_VERSIONS`. - -## Related Specs - -- [Pure E/B](pure_eb.md) — uses semi-analytic covariance propagation -- [COSEBIS](cosebis.md) — covariance propagation for bandpowers -- [Covariance Blind Consistency](covariance_blind_consistency.md) — validates diagonal agreement across blinds diff --git a/papers/bmodes/config/covariance_blind_consistency.md b/papers/bmodes/config/covariance_blind_consistency.md deleted file mode 100644 index 0e9ac650..00000000 --- a/papers/bmodes/config/covariance_blind_consistency.md +++ /dev/null @@ -1,33 +0,0 @@ -# Covariance Blind Consistency - -Method: [Covariance](covariance.md) - -## Claim - -Reporting covariance diagonals are consistent across blinds A, B, and C. Differences arise only from n(z) variations between blinds; survey geometry and cosmology are identical. - -## Evidence - -Diagonal ratios relative to blind A: -- `diag(Cov_B) / diag(Cov_A)` -- `diag(Cov_C) / diag(Cov_A)` - -Computed separately for ξ+ and ξ- blocks. - -**Metrics:** -- Max absolute deviation from unity -- Mean absolute deviation -- Pass/fail at 1% and 10% thresholds - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Version | `fiducial.version` | -| Scale range | `fiducial.min_sep` to `fiducial.max_sep` | -| Bins | `fiducial.nbins` | - -## Outputs - -- `evidence.json` — ratios, deviations, pass/fail flags -- `figure.png` — two-panel (ξ+, ξ-) ratio plot with ±1%, ±10% bands diff --git a/papers/bmodes/config/ecut_spec.md b/papers/bmodes/config/ecut_spec.md deleted file mode 100644 index 53041659..00000000 --- a/papers/bmodes/config/ecut_spec.md +++ /dev/null @@ -1,83 +0,0 @@ -This is your spec for a Ralph loop, a meditative iteration toward a desired state. - -## Desired State - -Three version comparison figures (pure E/B, COSEBIS, harmonic) showing B-modes for 4 -leak-corrected catalog versions side by side: - -- v1.4.6 (all galaxies) -- v1.4.6 with |e| < 0.7 -- v1.4.11.3 (all galaxies) -- v1.4.11.3 with |e| < 0.7 - -Each ecut version has its own covariance computed from the filtered catalog's n_eff and -sigma_e (with area inherited from the parent version). The full pipeline runs through -existing rules — ecut versions are just new entries in `config["versions"]` and -`cat_config.yaml`. - -**Done when:** -- `snakemake ecut_version_comparisons --dry-run` resolves cleanly (convenience target) -- Filtered catalogs exist at `results/ecut/SP_v1.4.{6,11.3}_ecut07.fits` -- `cat_config.yaml` has correct (non-zero) n_e and sigma_e for both ecut versions -- Existing version comparison rules accept a `{comparison}` wildcard in the output path - (e.g., `results/tapestry/{comparison}_pure_eb_version_comparison/`), with an input - function mapping `paper` → current versions, `ecut` → ecut versions -- `snakemake paper_pure_eb_version_comparison --dry-run` still works (backward compat via convenience target or alias) - -**Why:** Calum reports that |e| < 0.7 significantly reduces B-modes in v1.4.11 and v1.4.6, -but without trustable error bars. We need the full pipeline's covariances to quantify this. -DES-Y3 used e < 0.8 to remove stars. - -## Context - -### How versions flow through the pipeline - -Everything is driven by `config["versions"]` in `papers/bmodes/config/config.yaml`. Adding a -version there (plus its `cat_config.yaml` entry) makes it flow through all existing rules: -`xi`, `covariance`, `pure_eb_data_vector`, `cosebis_data_vector`, `cl_data_vector`. -The 2PCF and covariance are independent and can run in parallel. - -Key resolution functions in `workflow/Snakefile`: -- `get_shear_catalog()` — resolves catalog path from cat_config, strips `_leak_corr` -- `build_redshift_path()` — v1.4.11.x uses v1.4.6's n(z) -- `resolve_covariance_version()` — identity function (each version gets its own covariance) -- Wildcard constraint: `version=r"SP_v[\d.]+(_w_iv)?(_leak_corr)?"` — needs `_ecut\d+` - -Version comparison rules in `papers/bmodes/rules/claims.smk` (lines 131, 271, 386) use -`VERSIONS_LEAK_CORR` for inputs and derive version lists from config in the scripts. -These should be parameterized to accept a version list via `snakemake.params`, so the -same rules serve both paper and ecut comparisons. - -### Covariance depends on survey properties - -`get_cat_params()` in `workflow/rules/covariance.smk` (line 4) reads `cov_th.{A, n_e, sigma_e}` -from `cat_config.yaml`. These flow into CosmoCov INI files. Ecut versions need their own -values — same area as parent, but recomputed n_e and sigma_e. - -### Catalog filtering - -A preprocessing rule writes filtered FITS files (`results/ecut/`). Filter: -`sqrt(e1_leak_corrected² + e2_leak_corrected²) < 0.7`. Only `_leak_corr` versions — the -uncorrected columns can't be consistently filtered to guarantee the same rows. - -### Caveats - -- **n(z)**: Ellipticity cut preferentially removes faint/noisy objects. We reuse parent - n(z) — acceptable for B-mode null test, not for cosmological inference. -- **x_offsets**: Paper comparison has 4 versions with specific spacing. Ecut comparison - also has 4, so same offsets work. - -### Key files - -| What | Where | -|------|-------| -| Workflow config | `papers/bmodes/config/config.yaml` (search `ecut`) | -| Catalog config | `cosmo_val/cat_config.yaml` (search `ecut07`) | -| Pipeline orchestration | `workflow/Snakefile` (wildcard constraints, version resolution functions) | -| Version comparison rules | `papers/bmodes/rules/claims.smk` lines 131, 271, 386 | -| Version comparison scripts | `workflow/scripts/{pure_eb,cosebis,cl}_version_comparison.py` | -| Covariance params | `workflow/rules/covariance.smk` line 4 (`get_cat_params`) | - -## Skills - -`/snakemake` for DAG operations and job submission. diff --git a/papers/bmodes/config/harmonic_config_cosebis_comparison.md b/papers/bmodes/config/harmonic_config_cosebis_comparison.md deleted file mode 100644 index f3122e04..00000000 --- a/papers/bmodes/config/harmonic_config_cosebis_comparison.md +++ /dev/null @@ -1,69 +0,0 @@ -# Harmonic vs Configuration-Space COSEBIS Cross-Validation - -Cross-validate COSEBIS E_n and B_n modes computed from two independent paths. - -## Purpose - -Validate consistency between harmonic-space and configuration-space COSEBIS estimates. Both paths should yield the same E_n and B_n modes since they measure the same underlying shear field. Disagreement would indicate a problem in one of the estimation pipelines. - -## Methods - -### Path 1: Harmonic (pseudo-C_ell -> COSEBIS) - -1. Pseudo-C_ell from NaMaster (powspace binning, `cl.cosebis_nbins` bins — default 96) -2. `COSEBIS.cosebis_from_Cell(ell, Cell_E, Cell_B, theta)` transforms to E_n, B_n -3. Covariance propagated from C_ell covariance via linear transform T: `Cov_COSEBIS = T @ Cov_Cell @ T^T` - -### Path 2: Configuration (xi_pm -> COSEBIS) - -1. Fine-binned 2PCF from TreeCorr (integration grid: min_sep_int to max_sep_int, nbins_int bins) -2. `sp_validation.b_modes.calculate_cosebis()` integrates xi_pm with COSEBIS filter functions -3. Covariance from `COSEBIS.cosebis_covariance_from_xipm_covariance()` applied to theoretical xi_pm covariance - -## Angular Range - -Parameterized by `{angular_range}` wildcard: -- **full**: `cosebis.theta_min` to `cosebis.theta_max` (1–250 arcmin) -- **fiducial**: `fiducial.fiducial_min_scale` to `fiducial.fiducial_max_scale` (12–83 arcmin) - -The rule runs once per angular range, producing separate evidence and figures. - -## Reliable Modes - -At 96-bin powspace, modes n = 1–8 are quantitatively reliable from the harmonic path (validated on GLASS mocks: E_n/config within 2% for modes 1–5, <1% at the dense-ell harmonic ceiling for modes 1–7). Higher modes (n > 8) are shown grayed out for completeness but excluded from PTE calculations. The root cause for mode 9+ is W_n(ell) numerical precision at ~10^-14 amplitudes, not binning. The reliable mode count depends on `cl.cosebis_nbins` (6 at 32 bins, 8 at 96+ bins). - -## B-mode PTEs - -Chi-squared PTEs are computed for B-modes using reliable modes only (modes 1–8 at 96-bin), from both paths independently: -- **Harmonic-space**: propagated from pseudo-C_ell Gaussian covariance via linear transform -- **Config-space**: from CosmoCov theoretical xi_pm covariance - -At fiducial scale cuts, both methods find B-modes consistent with zero. At full range, both fail due to the known small-scale contamination. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| n_modes | `fiducial.nmodes` | -| theta_min (full) | `cosebis.theta_min` | -| theta_max (full) | `cosebis.theta_max` | -| theta_min (fiducial) | `fiducial.fiducial_min_scale` | -| theta_max (fiducial) | `fiducial.fiducial_max_scale` | -| powspace_nbins | `cl.cosebis_nbins` (default 96; separate from `cl.n_ell_bins` used for BB PTEs) | - -## Figures - -Per angular range: -1. **Data vector** (`figure.png`): Fiducial catalog. E-modes (top) and B-modes (bottom, B_n/sigma_n). Config vs harmonic overlaid. Unreliable modes grayed. -2. **Version comparison** (`figure_versions.png`): All leak-corrected versions, B_n/sigma_n. -3. **Paper figure**: `harmonic_config_cosebis_{angular_range}.png` - -## Known Limitations - -- Cross-covariance between harmonic and configuration-space methods is unknown, so no formal chi2/PTE is computed on the difference between the two paths. -- At 96 bins, modes n > 8 remain unreliable due to W_n(ell) numerical precision limits (COSEBIS amplitudes ~10^-14 at high modes), not binning. This is a fundamental ceiling validated on GLASS mocks with dense integer-ell sampling. - -## Depends on - -- cosebis (COSEBIS methodology) -- cl (harmonic-space pseudo-Cl estimation) diff --git a/papers/bmodes/config/harmonic_space_pte_matrices.md b/papers/bmodes/config/harmonic_space_pte_matrices.md deleted file mode 100644 index 0f17b6fc..00000000 --- a/papers/bmodes/config/harmonic_space_pte_matrices.md +++ /dev/null @@ -1,46 +0,0 @@ -# Harmonic-Space PTE Matrices - -Depends: [Harmonic-Space Power Spectra](cl.md), [2D Plots](2d_plots.md) -Method: [Harmonic-Space Power Spectra](cl.md) -Plotting: [2D Plots](2d_plots.md) - -## Claim - -Harmonic-space B-mode PTEs are consistent with noise at fiducial multipole range for the fiducial catalog version. PTE heatmaps across all (ell_min, ell_max) combinations show where B-modes become significant. The appendix presents all catalog versions (from `config.versions`) for comparison. - -Fiducial scale cuts from `cl.fiducial_ell_min` and `cl.fiducial_ell_max`. Full B-mode test range spans all multipole bins present in the input pseudo-Cℓ file. - -## Blind Handling - -Uses fiducial blind from `config["fiducial"]["blind"]`. The C_ℓ^BB data vector is identical across blinds; covariances vary with blind via n(z)-dependent theoretical predictions. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Versions | `versions` | -| Fiducial version | `fiducial.version` | -| Fiducial multipole range | `cl.fiducial_ell_min`, `cl.fiducial_ell_max` | - -Note: Number of multipole bins and full multipole range are determined by the input pseudo-Cℓ file, not config. - -## Evidence - -Per-version statistics: - -| Metric | Description | -|--------|-------------| -| `{version}.role` | "fiducial" or "appendix" | -| `{version}.pte_at_fiducial` | C_l^BB PTE at fiducial multipole range | -| `{version}.pte_at_full_range` | C_l^BB PTE at full multipole range | -| `{version}.n_evaluated` | Number of (ell_min, ell_max) pairs | -| `{version}.ell_range` | [ell_min, ell_max] of full range | -| `{version}.fiducial_ell_range` | [ell_min, ell_max] of fiducial range | -| `{version}.n_ell_bins` | Number of multipole bins | - -## Outputs - -- `figure_fiducial.png` — PTE heatmap for fiducial version (main text) -- `figure_appendix.png` — N-panel composite for all versions from `config.versions` (appendix) - -Heatmaps use a discrete PTE colormap (`make_pte_colormap` from `plotting_utils.py`) with solid blue below 0.05, solid red above 0.95, and a gradient between. No contour overlays. Fiducial multipole range marked with a plain black-edged rectangle (no hatching). diff --git a/papers/bmodes/config/pure_eb.md b/papers/bmodes/config/pure_eb.md deleted file mode 100644 index 6800b149..00000000 --- a/papers/bmodes/config/pure_eb.md +++ /dev/null @@ -1,46 +0,0 @@ -# Pure E/B Mode Decomposition - -Schneider et al. decomposition of shear correlation functions into pure E-mode, B-mode, and ambiguous components. - -## Purpose - -Traditional ξ± mix E and B modes. Pure-mode decomposition cleanly separates: -- **E-modes**: Cosmological lensing signal -- **B-modes**: Should be zero for pure lensing; non-zero indicates systematics -- **Ambiguous modes**: Modes that cannot be uniquely assigned to E or B - -## Config References - -| Parameter | Config Key | Description | -|-----------|------------|-------------| -| ξ+ scale cut | `fiducial.fiducial_xip_scale_cut` | Angular range for ξ+ | -| ξ- scale cut | `fiducial.fiducial_xim_scale_cut` | Angular range for ξ- | -| version | `fiducial.version` | Catalog version | - -## Method - -Following Schneider et al. (2002), decompose: -- ξ+(θ) → ξ+^E(θ) + ξ+^B(θ) + ξ+^amb(θ) -- ξ-(θ) → ξ-^E(θ) + ξ-^B(θ) + ξ-^amb(θ) - -Uses semi-analytical covariance propagation through the decomposition. - -## Data Products - -Precomputed decomposition stored in: -`results/paper_plots/intermediate/{version}_{blind}_pure_eb_semianalytic.npz` - -Uses fiducial blind (A) from config. Each NPZ contains: -- `theta`: Angular bins -- `xip_E`, `xim_E`: Pure E-mode components -- `xip_B`, `xim_B`: Pure B-mode components -- `xip_amb`, `xim_amb`: Ambiguous components -- `cov_pure_eb`: Full covariance matrix for decomposed modes (MC propagation) - -## Plotting Conventions - -- Total ξ± shown with filled markers -- E-modes in teal (secondary, lower alpha) -- B-modes in crimson (primary, unfilled markers, full opacity) -- Ambiguous in purple (secondary, lower alpha) -- Fiducial scale range highlighted diff --git a/papers/bmodes/config/pure_eb_covariance.md b/papers/bmodes/config/pure_eb_covariance.md deleted file mode 100644 index 05453d23..00000000 --- a/papers/bmodes/config/pure_eb_covariance.md +++ /dev/null @@ -1,45 +0,0 @@ -# Pure E/B Covariance - -Depends: [Pure E/B](pure_eb.md), [Covariance](covariance.md), [2D Plots](2d_plots.md), [Pure E/B Data Vector](pure_eb_data_vector.md), [Pure E/B Version Comparison](pure_eb_version_comparison.md) -Method: [Pure E/B](pure_eb.md), [Covariance](covariance.md) -Plotting: [2D Plots](2d_plots.md) - -## Claim - -Pure E/B covariance blocks are well-conditioned; ill-conditioning is localized to ambiguous modes. The 6-block correlation structure validates the semi-analytic covariance propagation for B-mode tests. - -## Evidence - -Block-wise condition numbers for the 120×120 pure E/B covariance (6 blocks of 20 bins each for ξ+/ξ- × E/B/amb): - -| Block | Description | Condition Number | -|-------|-------------|------------------| -| ξ_E | ξ+^E and ξ-^E combined | ~10^5 (well-conditioned) | -| ξ_B | ξ+^B and ξ-^B combined | ~10^5 (well-conditioned) | -| ξ_amb | ξ+^amb and ξ-^amb combined | ~10^15 (ill-conditioned) | - -**Key metrics:** -- Full matrix positive definite -- E and B blocks stable for PTE calculation -- Ill-conditioning confined to ambiguous modes (expected) - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Version | `fiducial.version` | -| Blind | `fiducial.blind` | -| Integration bins | `fiducial.nbins_int` | -| Reporting bins | `fiducial.nbins` | - -## Outputs - -- `pure_eb_covariance.png` — 6-block correlation matrix heatmap (icefire diverging colormap) -- `evidence.json` — condition numbers, eigenvalue bounds, positive definiteness - -## Visualization - -Correlation matrix with: -- icefire diverging colormap (−1 to +1) -- Block boundaries marked -- Labels for E/B/amb blocks diff --git a/papers/bmodes/config/pure_eb_data_vector.md b/papers/bmodes/config/pure_eb_data_vector.md deleted file mode 100644 index 6e334d9f..00000000 --- a/papers/bmodes/config/pure_eb_data_vector.md +++ /dev/null @@ -1,56 +0,0 @@ -# Pure E/B Data Vector - -Depends: [Pure E/B](pure_eb.md), [Pure E/B Covariance](pure_eb_covariance.md), [1D Plots](1d_plots.md) -Method: [Pure E/B](pure_eb.md) -Covariance: [Pure E/B Covariance](pure_eb_covariance.md) -Plotting: [1D Plots](1d_plots.md) - -## Claim - -B-mode signals in UNIONS cosmic shear are consistent with zero at fiducial scale cuts, validating the absence of significant systematic contamination in the shear measurement pipeline. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Fiducial version | `fiducial.version` | -| xi+ scale cut | `fiducial.fiducial_xip_scale_cut` | -| xi- scale cut | `fiducial.fiducial_xim_scale_cut` | -| PTE range | `statistics.pte_healthy_range` | - -## Evidence - -PTE values for B-mode null tests at two scale ranges, using fiducial blind (A): - -1. **Fiducial scale cuts**: Angular range used for cosmological inference -2. **Full theta range**: All measured angular bins (no cuts) - -For each range, report: -- xi+^B PTE -- xi-^B PTE -- Joint [xi+^B, xi-^B] PTE using full cross-covariance between components - -Fiducial PTEs should fall within healthy range for null hypothesis consistency. Full-range PTEs may show tension at scales excluded from analysis. - -| Metric | Description | -|--------|-------------| -| `fiducial.pte_xip_B` | xi+ B-mode PTE at fiducial cuts | -| `fiducial.pte_xim_B` | xi- B-mode PTE at fiducial cuts | -| `fiducial.pte_joint` | Joint PTE at fiducial cuts | -| `full.pte_xip_B` | xi+ B-mode PTE, full range | -| `full.pte_xim_B` | xi- B-mode PTE, full range | -| `full.pte_joint` | Joint PTE, full range | - -## Outputs - -**Main figure (leak-corrected, paper):** -- `figure.png` — Pure E/B decomposition showing xi+^B and xi-^B consistent with zero -- 1x2 layout: left panel (ξ+), right panel (ξ-) -- Each panel shows all decomposition components stacked: total, E-modes, ambiguous, B-modes -- Color coding: total (black), E-modes (teal), ambiguous (purple), B-modes (crimson) -- Excluded scale regions shaded gray (outside fiducial range) - -**Companion figure (uncorrected, dashboard only):** -- `figure_uncorrected.png` — Same layout, using uncorrected shear measurements -- Labeled "(uncorrected)" in title -- Not included in paper, for validation/comparison purposes diff --git a/papers/bmodes/config/pure_eb_version_comparison.md b/papers/bmodes/config/pure_eb_version_comparison.md deleted file mode 100644 index 583ce554..00000000 --- a/papers/bmodes/config/pure_eb_version_comparison.md +++ /dev/null @@ -1,38 +0,0 @@ -# Pure E/B Version Comparison - -Depends: [Pure E/B](pure_eb.md), [1D Plots](1d_plots.md) -Method: [Pure E/B](pure_eb.md) -Plotting: [1D Plots](1d_plots.md) - -## Claim - -B-mode correlation functions $\xi_{\pm}^B$ are consistent with zero across catalog versions. Total correlation functions $\xi_{\pm}$ show cosmological signal stability. Data outside fiducial scale cuts displayed greyed out. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Versions | `versions` | -| Fiducial | `fiducial.version` | -| Scale cut (xi+) | `fiducial.fiducial_xip_scale_cut` | -| Scale cut (xi-) | `fiducial.fiducial_xim_scale_cut` | - -## Evidence - -This claim produces visualizations only. Statistical evidence (PTEs) is in [Config Space PTE Matrices](config_space_pte_matrices.md). - -| Metric | Description | -|--------|-------------| -| `scale_cuts` | Angular ranges (fiducial cuts shown; excluded data greyed) | -| `versions_plotted` | Catalog versions included | - -## Outputs - -Main figure shows leak_corr catalog versions from `config.versions` for catalog evolution comparison. - -- `figure.png` — Four-panel figure with asymmetric row heights (catalog evolution) - - Top row (2/3 height): $\xi_+$ (left), $\xi_-$ (right) as $\theta \xi \times 10^4$ - - Bottom row (1/3 height): $\xi_+^B / \sigma$ (left), $\xi_-^B / \sigma$ (right) - - Data outside fiducial scale cuts shown greyed out - - B-mode error bars are unity by construction (normalized) - - Legend labels from `config.version_labels` diff --git a/papers/bmodes/config/xi_cosmology_paper.md b/papers/bmodes/config/xi_cosmology_paper.md deleted file mode 100644 index 10acd58d..00000000 --- a/papers/bmodes/config/xi_cosmology_paper.md +++ /dev/null @@ -1,110 +0,0 @@ -# ξ Cosmology Paper B-mode Reporting - -Spec for B-mode validation reporting in the configuration-space cosmology paper (Goh et al., unions_2d_shear_xi). - -Depends on: [pure_eb_data_vector](pure_eb_data_vector.md), [cosebis_version_comparison](cosebis_version_comparison.md), [covariance_blind_consistency](covariance_blind_consistency.md) - -## Scope - -This spec defines what B-mode evidence appears in the config-space paper (Goh et al.). The B-modes paper (Daley et al.) contains the full version comparison and methodological details. - -## Reporting Choices - -### Catalog Version - -Report only the **fiducial catalog** (with leakage correction, see `fiducial.version` in config). Version comparisons belong in the B-modes paper. - -### COSEBIS Mode Count - -Use **n=6** modes, not n=20. Rationale: -- Fewer modes = more conservative test (less prone to noise fluctuations) -- n=6 captures the dominant B-mode signal at small scales -- Consistent with scale cuts that exclude small and large angular scales - -### Statistics to Report - -Report both full-range and fiducial scale cut PTEs for both statistics. All PTEs are the **minimum across blinds** (conservative). - -| Statistic | Macro | -|-----------|-------| -| COSEBIS full PTE | `\cosebisfullPte` | -| COSEBIS fiducial PTE | `\cosebisfiducialPte` | -| Joint ξ±^B full PTE | `\ebfullPte` | -| Joint ξ±^B fiducial PTE | `\ebfiducialPte` | - -Values read from `evidence.json` at build time; see `generate_paper_macros.py`. - -The joint test combines ξ+^B and ξ-^B using the full cross-covariance matrix. - -### Scale Cuts - -Scale cuts from config (`fiducial.fiducial_xip_scale_cut`, `fiducial.fiducial_xim_scale_cut`): - -| Correlation | Min (arcmin) | Max (arcmin) | -|-------------|--------------|--------------| -| ξ+^B | 12 | 83 | -| ξ-^B | 12 | 83 | -| COSEBIS | 12 | 83 | - -## Text Requirements - -The E/B mode section should: -1. State that significant B-modes appear at extreme scales (motivating scale cuts) -2. Report passing PTEs at fiducial scale cuts -3. Reference the B-modes paper for methodology and version comparison -4. Use auto-generated macros (never hardcode values) - -## Blinding and Covariance - -The cosmological parameters (Ωm, σ8) are blinded with three independent blinds (A, B, C). Blinding affects the theoretical ξ± predictions, which propagate into the CosmoCov semi-analytical covariance matrices. - -### Covariance Variation Between Blinds - -From `covariance_blind_consistency`: -- ξ+ covariance diagonals vary by up to **9%** between blinds -- ξ- covariance diagonals vary by up to **8.6%** between blinds -- All blinds pass the 10% consistency threshold - -### PTE Variation Between Blinds - -The covariance variations propagate into PTE estimates differently for the two statistics. Example values (illustrative, see evidence.json for current): - -**Pure E/B (fiducial scale cuts):** - -| Blind | ξ+^B PTE | ξ-^B PTE | Joint PTE | -|-------|----------|----------|-----------| -| A | 0.48 | 0.10 | 0.28 | -| B | 0.49 | 0.10 | 0.31 | -| C | 0.50 | 0.08 | 0.29 | -| **Δ** | ~0.02 | ~0.02 | ~0.03 | - -**COSEBIS (fiducial scale cuts, n=6):** - -| Blind | PTE | -|-------|-----| -| A, B, C | ~0.29 | -| **Δ** | <0.001 | - -The near-identical COSEBIS PTEs reflect the compressed information in mode space — the integration over angular scales averages out the blind-dependent covariance variations. Pure E/B PTEs show more sensitivity (ΔPTE ≈ 0.03) because the test operates directly on angular bins where covariance differences are localized. - -### Reporting Strategy - -Report the **minimum PTE across blinds** as the conservative estimate. This ensures reported PTEs remain valid regardless of which blind is eventually unblinded. - -**Additional macros for blinding discussion:** - -| Macro | Description | -|-------|-------------| -| `\covXipMaxDev` | Max ξ+ covariance deviation between blinds | -| `\covXimMaxDev` | Max ξ- covariance deviation between blinds | -| `\ebJointPteDelta` | Joint PTE variation across blinds (max − min) | - -Values read from `covariance_blind_consistency/evidence.json` at build time. - -## Config References - -| Parameter | Config Key | -|-----------|------------| -| Fiducial version | `fiducial.version` | -| COSEBIS modes | `cosebis.mode_subsets` (use n=6 subset) | -| Scale cuts | `fiducial.fiducial_min_scale`, `fiducial.fiducial_max_scale` | diff --git a/papers/bmodes/rules/ecut.smk b/papers/bmodes/rules/ecut.smk deleted file mode 100644 index 9f941534..00000000 --- a/papers/bmodes/rules/ecut.smk +++ /dev/null @@ -1,218 +0,0 @@ -# workflow/rules/ecut.smk -""" -Ellipticity cut investigation: filter catalogs by |e| < threshold, -recompute survey properties, and run through the full B-mode pipeline. -""" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Configuration -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -ECUT_CONFIG = config.get("ecut", {}) -ECUT_VERSIONS = ECUT_CONFIG.get("versions_for_comparison", []) -ECUT_VERSION_LABELS = ECUT_CONFIG.get("version_labels", {}) -ECUT_FIDUCIAL = ECUT_CONFIG.get("fiducial_for_comparison", config["fiducial"]["version"]) - -# Parent version mapping: ecut version → parent version for catalog source -# e.g., SP_v1.4.6_ecut07 → SP_v1.4.6 -ECUT_PARENT_VERSIONS = { - ver: ver.split("_ecut")[0] - for ver in config.get("versions", []) - if "_ecut" in ver and "_leak_corr" not in ver -} - - -def _ecut_parent_catalog(wildcards): - """Resolve parent catalog path for an ecut version.""" - parent = ECUT_PARENT_VERSIONS.get(wildcards.version) - if parent is None: - raise ValueError(f"No parent version found for {wildcards.version}") - cat_config = config[parent] - shear_path = cat_config["shear"]["path"] - if shear_path.startswith("/"): - return shear_path - return str(Path(cat_config.get("subdir", "")) / shear_path) - - -def _ecut_parent_area(version): - """Get parent version's area for an ecut version.""" - base = version.replace("_leak_corr", "") - parent = ECUT_PARENT_VERSIONS.get(base, base) - return config[parent]["cov_th"]["A"] - - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Catalog filtering -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -rule filter_catalog_ellipticity: - """Filter a shear catalog by |e_leak_corrected| < threshold.""" - input: - catalog=_ecut_parent_catalog, - params: - e_max=lambda w: float(w.version.split("ecut")[1]) / 10, # ecut07 → 0.7 - e1_col="e1_leak_corrected", - e2_col="e2_leak_corrected", - w_col="w_des", - parent_area_deg2=lambda w: _ecut_parent_area(w.version), - output: - catalog="results/ecut/{version}.fits", - survey_props="results/ecut/{version}_survey_props.json", - wildcard_constraints: - version=r"SP_v[\d.]+_ecut\d+", - resources: - mem_mb=16000, - script: - "../scripts/filter_catalog_ellipticity.py" - - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Version comparison figures (parameterized by {comparison} wildcard) -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -# Comparison sets: maps {comparison} wildcard → (versions, labels, fiducial) -COMPARISON_SETS = { - "paper": { - "versions": VERSIONS_LEAK_CORR, - "labels": VERSION_LABELS, - "fiducial": config["plotting"].get("fiducial_for_comparison", config["fiducial"]["version"]), - }, - "ecut": { - "versions": ECUT_VERSIONS, - "labels": ECUT_VERSION_LABELS, - "fiducial": ECUT_FIDUCIAL, - }, -} - - -def _comparison_versions(comparison): - return COMPARISON_SETS[comparison]["versions"] - - -def _comparison_labels(comparison): - return COMPARISON_SETS[comparison]["labels"] - - -def _comparison_fiducial(comparison): - return COMPARISON_SETS[comparison]["fiducial"] - - -# --- Pure E/B version comparison --- - -def _pure_eb_comparison_inputs(wildcards): - versions = _comparison_versions(wildcards.comparison) - return { - "specs": [ - f"{CONFIG_DIR}/pure_eb_version_comparison.md", - f"{CONFIG_DIR}/pure_eb.md", - f"{CONFIG_DIR}/1d_plots.md", - ], - "config": f"{CONFIG_DIR}/config.yaml", - "pure_eb_data": [ - f"results/paper_plots/intermediate/{ver}_A_pure_eb_semianalytic.npz" - for ver in versions - ], - } - - -rule pure_eb_comparison: - """B-mode visualization: Pure E/B across catalog versions (parameterized).""" - input: - unpack(_pure_eb_comparison_inputs), - params: - version_labels=lambda w: _comparison_labels(w.comparison), - versions=lambda w: _comparison_versions(w.comparison), - fiducial_for_comparison=lambda w: _comparison_fiducial(w.comparison), - output: - evidence=f"{TAPESTRY_DIR}/{{comparison}}_pure_eb_version_comparison/evidence.json", - figure=f"{TAPESTRY_DIR}/{{comparison}}_pure_eb_version_comparison/figure.png", - paper_figure=f"{TAPESTRY_DIR}/{{comparison}}_pure_eb_version_comparison/paper_figure.pdf", - wildcard_constraints: - comparison=r"(paper|ecut)", - script: - "../scripts/pure_eb_version_comparison.py" - - -# --- COSEBIS version comparison --- - -def _cosebis_comparison_inputs(wildcards): - versions = _comparison_versions(wildcards.comparison) - return { - "specs": [ - f"{CONFIG_DIR}/cosebis_version_comparison.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/1d_plots.md", - ], - "config": f"{CONFIG_DIR}/config.yaml", - "xi_integration": [_xi_integration_path(ver) for ver in versions], - "cov_integration": [_cov_integration_path(ver, "A") for ver in versions], - } - - -rule cosebis_comparison: - """B-mode visualization: COSEBIS across catalog versions (parameterized).""" - input: - unpack(_cosebis_comparison_inputs), - params: - version_labels=lambda w: _comparison_labels(w.comparison), - versions=lambda w: _comparison_versions(w.comparison), - fiducial_for_comparison=lambda w: _comparison_fiducial(w.comparison), - output: - evidence=f"{TAPESTRY_DIR}/{{comparison}}_cosebis_version_comparison/evidence.json", - figure_stacked=f"{TAPESTRY_DIR}/{{comparison}}_cosebis_version_comparison/figure_stacked.png", - paper_stacked=f"{TAPESTRY_DIR}/{{comparison}}_cosebis_version_comparison/paper_figure.pdf", - wildcard_constraints: - comparison=r"(paper|ecut)", - script: - "../scripts/cosebis_version_comparison.py" - - -# --- Cl version comparison --- - -def _cl_comparison_inputs(wildcards): - versions = _comparison_versions(wildcards.comparison) - return { - "specs": [ - f"{CONFIG_DIR}/cl_version_comparison.md", - f"{CONFIG_DIR}/cl.md", - f"{CONFIG_DIR}/cl_data_vector.md", - ], - "config": f"{CONFIG_DIR}/config.yaml", - "pseudo_cl": [_pseudo_cl_path(ver) for ver in versions], - "pseudo_cl_cov": [_pseudo_cl_cov_path(ver) for ver in versions], - } - - -rule cl_comparison: - """C_ell^BB version comparison (parameterized).""" - input: - unpack(_cl_comparison_inputs), - params: - version_labels=lambda w: _comparison_labels(w.comparison), - versions=lambda w: _comparison_versions(w.comparison), - fiducial_for_comparison=lambda w: _comparison_fiducial(w.comparison), - ell_min_cut=config["cl"]["fiducial_ell_min"], - ell_max_cut=config["cl"]["fiducial_ell_max"], - output: - evidence=f"{TAPESTRY_DIR}/{{comparison}}_cl_version_comparison/evidence.json", - figure=f"{TAPESTRY_DIR}/{{comparison}}_cl_version_comparison/figure.png", - paper_figure=f"{TAPESTRY_DIR}/{{comparison}}_cl_version_comparison/paper_figure.pdf", - wildcard_constraints: - comparison=r"(paper|ecut)", - script: - "../scripts/cl_version_comparison.py" - - -localrules: pure_eb_comparison, cosebis_comparison, cl_comparison - - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Convenience targets -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -rule ecut_version_comparisons: - """All three ecut version comparison figures.""" - input: - rules.pure_eb_comparison.output[0].format(comparison="ecut"), - rules.cosebis_comparison.output[0].format(comparison="ecut"), - rules.cl_comparison.output[0].format(comparison="ecut"), diff --git a/papers/bmodes/rules/claims.smk b/papers/bmodes/rules/figures.smk similarity index 85% rename from papers/bmodes/rules/claims.smk rename to papers/bmodes/rules/figures.smk index 85891b25..322423d7 100644 --- a/papers/bmodes/rules/claims.smk +++ b/papers/bmodes/rules/figures.smk @@ -1,14 +1,14 @@ -# workflow/rules/claims.smk """ -Claims — testable assertions that produce evidence. -Claims depend on methods (for technique definitions) and compute outputs (for data). +Paper figures. Each rule plots from compute-workflow outputs and writes an +evidence.json of summary statistics (PTEs, chi2) that paper.smk turns into +LaTeX macros and tables. """ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # Configuration # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# CONFIG_DIR, TAPESTRY_DIR, PAPER_FIGURES_DIR, BLINDS, FIDUCIAL, PLANCK18 defined in Snakefile +# TAPESTRY_DIR, PAPER_FIGURES_DIR, BLINDS, FIDUCIAL, PLANCK18 defined in Snakefile # COSMO_VAL, COSMO_INFERENCE, covariance_path() defined in Snakefile COSMO_VAL_OUTPUT = str(COSMO_VAL) # String version for f-string interpolation @@ -33,7 +33,7 @@ MOCK_VERSION = f"{FIDUCIAL['mock_version']}_leak_corr" # Filter versions for different analysis types # Pure E/B and PTEs only apply to leak-corrected versions -VERSIONS_LEAK_CORR = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] +VERSIONS_LEAK_CORR = [v for v in config["versions"] if "_leak_corr" in v] # Uncorrected counterparts (bare catalog, no leakage correction) VERSIONS_UNCORRECTED = [v.replace("_leak_corr", "") for v in VERSIONS_LEAK_CORR] @@ -52,7 +52,7 @@ def _extract_version_number(version_string): return match.group(1) if match else version_string -def _per_version_figure_outputs(claim_dir): +def _per_version_figure_outputs(fig_dir): """Generate output dict for 9 per-version figures. Returns dict mapping output keys to paths for all 9 figures: @@ -60,11 +60,11 @@ def _per_version_figure_outputs(claim_dir): - figure_v{X.Y.Z}.png for each leak-corrected version - figure_v{X.Y.Z}_uncorrected.png for each uncorrected version """ - outputs = {"figure": f"{claim_dir}/figure.png"} + outputs = {"figure": f"{fig_dir}/figure.png"} for ver_lc in sorted(VERSION_LABELS.keys(), key=lambda v: -len(v)): ver_num = _extract_version_number(ver_lc) - outputs[f"figure_{ver_num.replace('.', '_')}"] = f"{claim_dir}/figure_{ver_num}.png" - outputs[f"figure_{ver_num.replace('.', '_')}_uncorrected"] = f"{claim_dir}/figure_{ver_num}_uncorrected.png" + outputs[f"figure_{ver_num.replace('.', '_')}"] = f"{fig_dir}/figure_{ver_num}.png" + outputs[f"figure_{ver_num.replace('.', '_')}_uncorrected"] = f"{fig_dir}/figure_{ver_num}_uncorrected.png" return outputs @@ -130,7 +130,7 @@ def _pseudo_cl_cov_path(version, blind="A", nbins=32): # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# COSEBIS Claims +# COSEBIS # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ rule cosebis_version_comparison: @@ -139,12 +139,6 @@ rule cosebis_version_comparison: Plotting only - statistical PTEs are in cosebis_pte_matrix. """ input: - specs=[ - f"{CONFIG_DIR}/cosebis_version_comparison.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/1d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # COSEBIs only for leak-corrected versions xi_integration=[_xi_integration_path(ver) for ver in VERSIONS_LEAK_CORR], cov_integration=[_cov_integration_path(ver, "A") for ver in VERSIONS_LEAK_CORR], @@ -171,12 +165,6 @@ rule cosebis_data_vector: - figure_v{X.Y.Z}_uncorrected.png: each version, uncorrected, with title """ input: - specs=[ - f"{CONFIG_DIR}/cosebis_data_vector.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/1d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: xi_{version} and cov_{version} for all versions **{f"xi_{ver}": _xi_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"cov_{ver}": _cov_integration_path(ver, "A") for ver in VERSIONS_ALL_FOR_PLOTS}, @@ -190,32 +178,8 @@ rule cosebis_data_vector: "../scripts/cosebis_data_vector.py" -rule cosebis_binning_comparison: - """COSEBIS angular binning convergence: 1,000 vs 10,000 ξ± bins. - - Tests whether the numerical integration of T±n(θ) × ξ±(θ) is converged - at 1,000 bins by comparing B_n values and PTEs against 10,000-bin results. - Uses the 1,000-bin COSEBIS covariance for both (integration-independent - if converged). Ref: Asgari et al. 2017. - """ - input: - xi_1k=_xi_integration_path(FIDUCIAL_VERSION), - xi_10k=( - f"{COSMO_VAL_OUTPUT}/{FIDUCIAL_VERSION}_xi_minsep={FIDUCIAL['min_sep_int']}" - f"_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch={FIDUCIAL['npatch']}.txt" - ), - cov_1k=_cov_integration_path(FIDUCIAL_VERSION, "A"), - output: - evidence=f"{TAPESTRY_DIR}/cosebis_binning_comparison/evidence.json", - figure=f"{TAPESTRY_DIR}/cosebis_binning_comparison/figure.png", - resources: - mem_mb=8000, - script: - "../scripts/cosebis_binning_comparison.py" - - # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Pure E/B Claims +# Pure E/B # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # Number of parallel chunks for MC covariance estimation @@ -247,7 +211,7 @@ rule precompute_pure_eb_chunk: rule precompute_pure_eb: """Gather MC sample chunks and compute final pure E/B covariance.""" wildcard_constraints: - version=r"[^_]+_v[\d.]+(_ecut\d+)?(_leak_corr)?", # e.g. SP_v1.4.6, SP_v1.4.6_ecut07_leak_corr + version=r"[^_]+_v[\d.]+(_leak_corr)?", # e.g. SP_v1.4.6.3, SP_v1.4.6.3_leak_corr blind=r"[ABC]", input: chunks=expand( @@ -279,12 +243,6 @@ rule pure_eb_data_vector: - figure_v{X.Y.Z}_uncorrected.png: each version, uncorrected, with title """ input: - specs=[ - f"{CONFIG_DIR}/pure_eb_data_vector.md", - f"{CONFIG_DIR}/pure_eb.md", - f"{CONFIG_DIR}/1d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: pure_eb_{version} and cov_{version} for all versions **{f"pure_eb_{ver}": f"results/paper_plots/intermediate/{ver}_{FIDUCIAL['blind']}_pure_eb_semianalytic.npz" for ver in VERSIONS_ALL_FOR_PLOTS}, @@ -305,12 +263,6 @@ rule pure_eb_version_comparison: Uses E-mode errors from pure_eb covariance as proxy for total xi (E dominates). """ input: - specs=[ - f"{CONFIG_DIR}/pure_eb_version_comparison.md", - f"{CONFIG_DIR}/pure_eb.md", - f"{CONFIG_DIR}/1d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # Pure E/B only for leak-corrected versions pure_eb_data=[ f"results/paper_plots/intermediate/{ver}_A_pure_eb_semianalytic.npz" @@ -338,13 +290,6 @@ rule pure_eb_covariance: Uses blind A covariance for visualization (structure is similar across blinds). """ input: - specs=[ - f"{CONFIG_DIR}/pure_eb_covariance.md", - f"{CONFIG_DIR}/pure_eb.md", - f"{CONFIG_DIR}/covariance.md", - f"{CONFIG_DIR}/2d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", pure_eb_data=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_A_pure_eb_semianalytic.npz", output: evidence=f"{TAPESTRY_DIR}/pure_eb_covariance/evidence.json", @@ -379,7 +324,7 @@ rule calculate_pure_eb_ptes: # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Harmonic-Space Claims (Cl) +# Harmonic space (Cl) # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ rule cl_data_vector: @@ -391,11 +336,6 @@ rule cl_data_vector: - figure_v{X.Y.Z}_uncorrected.png: each version, uncorrected, with title """ input: - specs=[ - f"{CONFIG_DIR}/cl_data_vector.md", - f"{CONFIG_DIR}/cl.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # Per-version inputs: pseudo_cl_{version} and pseudo_cl_cov_{version} for all versions **{f"pseudo_cl_{ver}": _pseudo_cl_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"pseudo_cl_cov_{ver}": _pseudo_cl_cov_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, @@ -413,12 +353,6 @@ rule cl_data_vector: rule cl_version_comparison: """C_ell^BB version comparison across catalog versions.""" input: - specs=[ - f"{CONFIG_DIR}/cl_version_comparison.md", - f"{CONFIG_DIR}/cl.md", - f"{CONFIG_DIR}/cl_data_vector.md", - ], - config=f"{CONFIG_DIR}/config.yaml", cl_data_vector_evidence=rules.cl_data_vector.output.evidence, # Cl version comparison only for leak-corrected versions pseudo_cl=[_pseudo_cl_path(ver) for ver in VERSIONS_LEAK_CORR], @@ -472,14 +406,6 @@ rule config_space_pte_matrices: Appendix: 3x3 composite for all versions (3 rows x 3 statistics) """ input: - specs=[ - f"{CONFIG_DIR}/config_space_pte_matrices.md", - f"{CONFIG_DIR}/pure_eb.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/2d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", - # Claim dependencies pure_eb_data_vector=f"{TAPESTRY_DIR}/pure_eb_data_vector/evidence.json", cosebis_data_vector=f"{TAPESTRY_DIR}/cosebis_data_vector/evidence.json", # Data inputs (fiducial blind only) @@ -512,12 +438,6 @@ rule harmonic_space_pte_matrices: Uses fiducial blind covariance (blind independence validated in bb_covariance_blind_independence). """ input: - specs=[ - f"{CONFIG_DIR}/harmonic_space_pte_matrices.md", - f"{CONFIG_DIR}/cl.md", - f"{CONFIG_DIR}/2d_plots.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # Harmonic PTE matrices for both corrected and uncorrected versions pseudo_cl=[_pseudo_cl_path(ver) for ver in VERSIONS_CONFIG_SPACE_PTES], pseudo_cl_cov=[ @@ -549,14 +469,6 @@ rule bb_covariance_blind_independence: for the fiducial version. """ input: - specs=[ - f"{CONFIG_DIR}/bb_covariance_blind_independence.md", - f"{CONFIG_DIR}/covariance.md", - f"{CONFIG_DIR}/pure_eb.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/cl.md", - ], - config=f"{CONFIG_DIR}/config.yaml", # Per-blind MC-propagated pure E/B covariances (using mock_version for all blinds) **{f"pure_eb_{b}": f"results/paper_plots/intermediate/{MOCK_VERSION}_{b}_pure_eb_semianalytic.npz" for b in BLINDS}, @@ -603,12 +515,6 @@ rule harmonic_config_cosebis_comparison: wildcard_constraints: angular_range="full|fiducial", input: - specs=[ - f"{CONFIG_DIR}/harmonic_config_cosebis_comparison.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/cl.md", - ], - config=f"{CONFIG_DIR}/config.yaml", **{f"pseudo_cl_{ver}": _pseudo_cl_path(ver, nbins=_COSEBIS_NBINS) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"pseudo_cl_cov_{ver}": _pseudo_cl_cov_path(ver, nbins=_COSEBIS_NBINS) for ver in VERSIONS_ALL_FOR_PLOTS}, **{f"xi_{ver}": _xi_integration_path(ver) for ver in VERSIONS_ALL_FOR_PLOTS}, @@ -635,12 +541,6 @@ rule cosebis_filter_overlay: Shows why coarse bandpowers underresolve higher COSEBIS modes. """ input: - specs=[ - f"{CONFIG_DIR}/cosebis_filter_overlay.md", - f"{CONFIG_DIR}/cosebis.md", - f"{CONFIG_DIR}/cl.md", - ], - config=f"{CONFIG_DIR}/config.yaml", pseudo_cl=_pseudo_cl_path(FIDUCIAL_VERSION), pseudo_cl_cov=_pseudo_cl_cov_path(FIDUCIAL_VERSION), output: diff --git a/papers/bmodes/rules/paper.smk b/papers/bmodes/rules/paper.smk new file mode 100644 index 00000000..ead4e1d3 --- /dev/null +++ b/papers/bmodes/rules/paper.smk @@ -0,0 +1,70 @@ +""" +Paper outputs: LaTeX macros and PTE tables built from the figure rules' +evidence.json summaries, and the aggregate `paper` target. +""" + +# Figure rules whose outputs the paper target builds +FIGURE_RULES = [ + "cosebis_version_comparison", + "cosebis_data_vector", + "pure_eb_data_vector", + "pure_eb_version_comparison", + "pure_eb_covariance", + "cl_data_vector", + "cl_version_comparison", + "config_space_pte_matrices", + "harmonic_space_pte_matrices", + "bb_covariance_blind_independence", + "cosebis_filter_overlay", +] + +_HARMONIC_COSEBIS_ANGULAR_RANGES = ["full", "fiducial"] + + +localrules: xi_cosmology_paper_macros, paper_macros, paper + + +rule xi_cosmology_paper_macros: + """B-mode macros for the configuration-space cosmology paper (Goh et al.): + fiducial version, n=6 COSEBIS, joint pure-mode PTEs at full and fiducial + scales.""" + input: + cosebis_evidence=rules.cosebis_version_comparison.output.evidence, + pure_eb_evidence=rules.pure_eb_data_vector.output.evidence, + bb_blind_evidence=rules.bb_covariance_blind_independence.output.evidence, + output: + macros="docs/unions_release/unions_2d_shear_xi/claims_macros.tex", + params: + tapestry_dir=TAPESTRY_DIR, + script: + "../scripts/generate_paper_macros.py" + + +rule paper_macros: + """LaTeX macros and PTE tables for the B-modes paper (Daley et al.).""" + input: + cosebis_evidence=rules.cosebis_version_comparison.output.evidence, + pure_eb_evidence=rules.pure_eb_data_vector.output.evidence, + pure_eb_covariance=rules.pure_eb_covariance.output.evidence, + config_space_pte=rules.config_space_pte_matrices.output.evidence, + harmonic_space_pte=rules.harmonic_space_pte_matrices.output.evidence, + output: + bmodes="docs/unions_release/unions_bmodes/claims_macros.tex", + pte_table_results="docs/unions_release/unions_bmodes/pte_table_results.tex", + pte_table_appendix="docs/unions_release/unions_bmodes/pte_table_appendix.tex", + params: + tapestry_dir=TAPESTRY_DIR, + script: + "../scripts/generate_paper_macros.py" + + +rule paper: + """Every paper figure, macro file and PTE table.""" + input: + rules.xi_cosmology_paper_macros.output, + rules.paper_macros.output, + expand( + rules.harmonic_config_cosebis_comparison.output, + angular_range=_HARMONIC_COSEBIS_ANGULAR_RANGES, + ), + *(getattr(rules, name).output for name in FIGURE_RULES), diff --git a/papers/bmodes/rules/presentation.smk b/papers/bmodes/rules/presentation.smk deleted file mode 100644 index c59f0aa3..00000000 --- a/papers/bmodes/rules/presentation.smk +++ /dev/null @@ -1,155 +0,0 @@ -"""Snakemake rules for Moriond 2026 talk figures. - -Converts paper-repo PDFs to PNGs for the reveal.js deck. Whenever a -collaborator updates a figure in their paper directory, re-running -``snakemake talk_figures`` picks up the change automatically. -""" - -TALK_DIR = "docs/talks/26_Moriond_UNIONS" -PAPER_III = "docs/unions_release/unions_bmodes/Figures" -PAPER_IV = "docs/unions_release/unions_2d_shear_xi/Figures" -PAPER_V = "docs/unions_release/unions_harmonic/Figures" - -# output name (without .png) → source PDF -TALK_FIGURES = { - # Paper III — B-mode data vectors - "pure_eb_data_vector": f"{PAPER_III}/pure_eb_data_vector.pdf", - "cosebis_data_vector": f"{PAPER_III}/cosebis_data_vector.pdf", - "cl_data_vector": f"{PAPER_III}/cl_data_vector.pdf", - "config_space_pte_fiducial": f"{PAPER_III}/config_space_pte_fiducial.pdf", - "harmonic_config_cosebis_full": f"{PAPER_III}/harmonic_config_cosebis_full.pdf", - "harmonic_config_cosebis_fiducial": f"{PAPER_III}/harmonic_config_cosebis_fiducial.pdf", - # Paper IV — configuration-space cosmology - "best_fit_xipm": f"{PAPER_IV}/best_fit_xipm_SP_v1.4.6.3_B.pdf", - "contour_survey_comparison": f"{PAPER_IV}/SP_v1.4.6.3_B_fiducial_config_contour_plot.pdf", - "whisker_plot": f"{PAPER_IV}/S8_whisker_plot.pdf", - "Omega_m_sigma_8_joint_config": f"{PAPER_IV}/Omega_m_sigma_8_joint_config.pdf", - "S8_comparison_config_harm": f"{PAPER_IV}/S8_comparison_config_harm.pdf", - # Paper V — harmonic-space cosmology - "Cell_EE_bestfit": f"{PAPER_V}/paperplot_Cell_EE_and_best_fit.pdf", - "S8_joint_config_harm": f"{PAPER_V}/S8_joint_config_harm.pdf", - "S8_difference_config_harm": f"{PAPER_V}/S8_difference_config_harm_map_2D.pdf", - "contours_blind_harmonic": f"{PAPER_V}/cosmological_constraints_blind/contours_s8_omegam_cell.pdf", - "contours_config_vs_harmonic": f"{PAPER_V}/cosmological_constraints_cl_vs_xi/contours_s8_omegam_cell.pdf", - "contours_weak_lensing_surveys": f"{PAPER_V}/cosmological_constraints_weak_lensing/contours_OMEGA_M_S8_2D_weak_lensing_all_surveys.pdf", -} - - -localrules: talk_figure, talk_figure_preview, talk_figures, talk_previews, presentation_blind_nz_plot, presentation_s8_convergence, presentation_s8_with_unions, presentation_omega_m_difference, presentation_s8_scatter_mocks, presentation_pte_cosebis - -ruleorder: presentation_blind_nz_plot > talk_figure - - -rule talk_figures: - """Build all talk PNGs from paper-repo PDFs.""" - input: - expand(f"{TALK_DIR}/images/{{name}}.png", name=TALK_FIGURES.keys()), - f"{TALK_DIR}/images/blind_nz_ABC.png", - f"{TALK_DIR}/images/s8_convergence_whisker.png", - f"{TALK_DIR}/images/omega_m_difference_config_harm.png", - f"{TALK_DIR}/images/s8_scatter_config_vs_harmonic.png", - f"{TALK_DIR}/images/pte_cosebis_talk.png", - f"{TALK_DIR}/images/s8_convergence_with_unions.png", - - -rule talk_previews: - """Build low-res preview PNGs (< 1800px) safe for Claude to read.""" - input: - expand(f"{TALK_DIR}/images/preview/{{name}}.png", name=TALK_FIGURES.keys()), - - -rule talk_figure: - """Convert a single paper PDF to high-res PNG for the talk. - - `container: None` on purpose: ImageMagick's `convert` is a host tool and is - not installed in the sp_validation image. - """ - input: - pdf=lambda w: TALK_FIGURES[w.name], - output: - f"{TALK_DIR}/images/{{name}}.png", - wildcard_constraints: - name="|".join(TALK_FIGURES.keys()), - container: - None - shell: - "convert -density 300 {input.pdf} -quality 95 {output}" - - -rule talk_figure_preview: - """Downscale a talk figure to < 1800px for safe AI reading. - - `container: None` for the same reason as talk_figure: `convert` is a host - tool, absent from the image. - """ - input: - f"{TALK_DIR}/images/{{name}}.png", - output: - f"{TALK_DIR}/images/preview/{{name}}.png", - wildcard_constraints: - name="|".join(TALK_FIGURES.keys()), - container: - None - shell: - "convert {input} -resize '1800x>' {output}" - - -rule presentation_s8_convergence: - """S8 convergence whisker plot for Moriond slide 2.""" - output: - f"{TALK_DIR}/images/s8_convergence_whisker.png", - shell: - "python {TALK_DIR}/plot_s8_convergence.py" - - -rule presentation_s8_with_unions: - """S8 convergence whisker plot WITH UNIONS constraints for slide 14 animation.""" - output: - f"{TALK_DIR}/images/s8_convergence_with_unions.png", - shell: - "python {TALK_DIR}/plot_s8_with_unions.py" - - -rule presentation_blind_nz_plot: - """Plot all three blinded n(z) curves for Moriond presentation.""" - input: - nz_A=lambda w: build_redshift_path(FIDUCIAL["version"], "A"), - nz_B=lambda w: build_redshift_path(FIDUCIAL["version"], "B"), - nz_C=lambda w: build_redshift_path(FIDUCIAL["version"], "C"), - output: - f"{TALK_DIR}/images/blind_nz_ABC.png", - script: - "../scripts/plot_presentation_blind_nz.py" - - -rule presentation_omega_m_difference: - """Delta Omega_m histogram from 350 GLASS mocks (config vs harmonic).""" - input: - mock_summary="/n09data/guerrini/glass_mock_chains/summary_parameter_constraints_merged_v6.txt", - output: - f"{TALK_DIR}/images/omega_m_difference_config_harm.png", - shell: - "python {TALK_DIR}/plot_omega_m_difference.py" - - -rule presentation_s8_scatter_mocks: - """S8(config) vs S8(harmonic) scatter from 350 GLASS mocks with data crosshair.""" - input: - mock_summary="/n09data/guerrini/glass_mock_chains/summary_parameter_constraints_merged_v6.txt", - output: - f"{TALK_DIR}/images/s8_scatter_config_vs_harmonic.png", - shell: - "python {TALK_DIR}/plot_s8_scatter_mocks.py" - - -rule presentation_pte_cosebis: - """COSEBIS B_n PTE heatmap for Moriond talk (single panel, talk-sized).""" - input: - pte_files=[ - f"{TAPESTRY_DIR}/cosebis_pte_matrix/pte_values/{FIDUCIAL['version']}/{FIDUCIAL['blind']}/pte_{i:03d}_{j:03d}.json" - for i, j in PTE_SCALE_CUT_PAIRS - ], - output: - f"{TALK_DIR}/images/pte_cosebis_talk.png", - shell: - "python {TALK_DIR}/plot_pte_cosebis_talk.py" diff --git a/papers/bmodes/rules/synthesis.smk b/papers/bmodes/rules/synthesis.smk deleted file mode 100644 index 64b60770..00000000 --- a/papers/bmodes/rules/synthesis.smk +++ /dev/null @@ -1,141 +0,0 @@ -# workflow/rules/synthesis.smk -""" -Synthesis — paper specs and aggregate targets. -Synthesis rules aggregate claims into papers and generate outputs for publication. -""" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Configuration -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -# Variables from included files: CONFIG_DIR, TAPESTRY_DIR (Snakefile) - -# Claim rules that produce evidence.json — single source of truth for all_tapestry -# Each entry is a rule name; we access rules.X.output to get all outputs -CLAIM_RULES = [ - "cosebis_version_comparison", - "cosebis_data_vector", - "pure_eb_data_vector", - "pure_eb_version_comparison", - "pure_eb_covariance", - "cl_data_vector", - "cl_version_comparison", - "config_space_pte_matrices", - "harmonic_space_pte_matrices", - "bb_covariance_blind_independence", - "cosebis_filter_overlay", -] - -# Wildcard claim rules expanded over their parameter values -_HARMONIC_COSEBIS_ANGULAR_RANGES = ["full", "fiducial"] - - -def _claim_outputs(): - """Get all outputs from claim rules.""" - return {name: getattr(rules, name).output for name in CLAIM_RULES} - - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Paper Macros -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -localrules: xi_cosmology_paper, paper_macros, bmodes_paper_spec, all_tapestry - -rule xi_cosmology_paper: - """Spec for B-mode reporting in configuration-space paper (Goh et al.). - - Depends on COSEBIS version comparison, pure E/B data vector, and covariance consistency. - Reports fiducial version, n=6 COSEBIS, joint pure-mode PTEs at both full and fiducial scales. - Also generates evidence.json for dashboard dependency tracking. - """ - input: - spec=f"{CONFIG_DIR}/xi_cosmology_paper.md", - cosebis_evidence=rules.cosebis_version_comparison.output.evidence, - pure_eb_evidence=rules.pure_eb_data_vector.output.evidence, - bb_blind_evidence=rules.bb_covariance_blind_independence.output.evidence, - output: - macros="docs/unions_release/unions_2d_shear_xi/claims_macros.tex", - evidence=f"{TAPESTRY_DIR}/xi_cosmology_paper/evidence.json", - params: - tapestry_dir=TAPESTRY_DIR, - script: - "../scripts/generate_paper_macros.py" - - -rule paper_macros: - """Generate LaTeX macros and tables for B-modes paper (Daley et al.).""" - input: - cosebis_evidence=rules.cosebis_version_comparison.output.evidence, - pure_eb_evidence=rules.pure_eb_data_vector.output.evidence, - pure_eb_covariance=rules.pure_eb_covariance.output.evidence, - # PTE composite evidence for table generation - config_space_pte=rules.config_space_pte_matrices.output.evidence, - harmonic_space_pte=rules.harmonic_space_pte_matrices.output.evidence, - output: - bmodes="docs/unions_release/unions_bmodes/claims_macros.tex", - pte_table_results="docs/unions_release/unions_bmodes/pte_table_results.tex", - pte_table_appendix="docs/unions_release/unions_bmodes/pte_table_appendix.tex", - params: - tapestry_dir=TAPESTRY_DIR, - script: - "../scripts/generate_paper_macros.py" - - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Paper Spec -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -rule bmodes_paper_spec: - """Generate evidence.json for bmodes_paper spec. - - Dependencies include both evidence files and figure outputs to ensure - all stale plots are regenerated. - """ - input: - spec=f"{CONFIG_DIR}/bmodes_paper.md", - # Upstream evidence (using rules.X.output for single source of truth) - pure_eb_covariance=rules.pure_eb_covariance.output.evidence, - pure_eb_data_vector=rules.pure_eb_data_vector.output.evidence, - cosebis_data_vector=rules.cosebis_data_vector.output.evidence, - cosebis_version_comparison=rules.cosebis_version_comparison.output.evidence, - cl_data_vector=rules.cl_data_vector.output.evidence, - cl_version_comparison=rules.cl_version_comparison.output.evidence, - config_space_pte=rules.config_space_pte_matrices.output.evidence, - harmonic_space_pte=rules.harmonic_space_pte_matrices.output.evidence, - # Paper figure dependencies (ensures version comparison plots regenerate) - pure_eb_version_comparison=rules.pure_eb_version_comparison.output.evidence, - cosebis_bmode_stacked=rules.cosebis_version_comparison.output.paper_stacked, - # Consistency checks - bb_covariance_blind=rules.bb_covariance_blind_independence.output.evidence, - output: - evidence=f"{TAPESTRY_DIR}/bmodes_paper/evidence.json", - run: - import json - from datetime import datetime - from pathlib import Path - - evidence = { - "id": "bmodes_paper", - "generated": datetime.now().isoformat(), - "evidence": {"type": "synthesis"}, - "output": {}, - } - with open(output.evidence, "w") as f: - json.dump(evidence, f, indent=2) - - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# Aggregate Targets -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -rule all_tapestry: - """Aggregate target for all claim evidence and paper outputs.""" - input: - bmodes_paper=rules.bmodes_paper_spec.output, - xi_cosmology_paper=rules.xi_cosmology_paper.output, - paper_macros=rules.paper_macros.output, - harmonic_cosebis=expand( - f"{TAPESTRY_DIR}/harmonic_config_cosebis_comparison_{{angular_range}}/evidence.json", - angular_range=_HARMONIC_COSEBIS_ANGULAR_RANGES, - ), - **_claim_outputs(), diff --git a/papers/bmodes/scripts/bb_covariance_blind_independence.py b/papers/bmodes/scripts/bb_covariance_blind_independence.py index 256a79b1..c66607af 100644 --- a/papers/bmodes/scripts/bb_covariance_blind_independence.py +++ b/papers/bmodes/scripts/bb_covariance_blind_independence.py @@ -526,8 +526,6 @@ def main( # Build evidence evidence = { - "spec_id": "bb_covariance_blind_independence", - "spec_path": "workflow/config/bb_covariance_blind_independence.md", "depends_on": ["covariance", "pure_eb", "cosebis", "pseudo_cl"], "generated": datetime.now().isoformat(), "evidence": { diff --git a/papers/bmodes/scripts/cl_data_vector.py b/papers/bmodes/scripts/cl_data_vector.py index 4eeca960..e47c24d8 100644 --- a/papers/bmodes/scripts/cl_data_vector.py +++ b/papers/bmodes/scripts/cl_data_vector.py @@ -158,7 +158,7 @@ def _create_cl_figure( # --------------------------------------------------------------------------- # Canonical per-version file paths (COSMO_VAL results tree). The DAG read these -# via _pseudo_cl_path() in claims.smk; the CLI reconstructs them from --results-dir +# via _pseudo_cl_path() in figures.smk; the CLI reconstructs them from --results-dir # so the version sweep is self-contained (lc produces only the fiducial version). # --------------------------------------------------------------------------- def _pseudo_cl(results_dir, ver, blind="A", nbins=32): @@ -290,7 +290,6 @@ def main( ) evidence_data = { - "spec_id": "cl_data_vector", "generated": datetime.now().isoformat(), "evidence": { # Full range PTEs diff --git a/papers/bmodes/scripts/cl_version_comparison.py b/papers/bmodes/scripts/cl_version_comparison.py index f70a6977..c3c77ddd 100644 --- a/papers/bmodes/scripts/cl_version_comparison.py +++ b/papers/bmodes/scripts/cl_version_comparison.py @@ -86,7 +86,7 @@ def main( plotting_config = config["plotting"] version_labels = plotting_config["version_labels"] - # Leak-corrected, non-ecut versions (matches VERSIONS_LEAK_CORR in claims.smk) + # Leak-corrected versions (matches VERSIONS_LEAK_CORR in figures.smk) versions = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] # Which version gets the fiducial reference line in boxes @@ -348,7 +348,6 @@ def main( evidence_versions[f"{v}_dof_eb_cut"] = int(data["dof_eb_cut"]) evidence_data = { - "spec_id": "cl_version_comparison", "generated": datetime.now().isoformat(), "evidence": { "versions": evidence_versions, diff --git a/papers/bmodes/scripts/compute_cosebis_pte_single.py b/papers/bmodes/scripts/compute_cosebis_pte_single.py index 452fc514..73563a8c 100644 --- a/papers/bmodes/scripts/compute_cosebis_pte_single.py +++ b/papers/bmodes/scripts/compute_cosebis_pte_single.py @@ -23,7 +23,7 @@ def _pte_scale_cut_pairs(): """(i_min, i_max) index pairs for the PTE matrix, excluding the polynomial- - root-unstable subsets. Mirrors _pte_scale_cut_pairs() in claims.smk.""" + root-unstable subsets. Mirrors _pte_scale_cut_pairs() in figures.smk.""" unstable = {(9, 10), (10, 11), (11, 12), (13, 14)} return [ (i, j) for i in range(20) for j in range(i + 1, 21) if (i, j) not in unstable diff --git a/papers/bmodes/scripts/config_space_pte_matrices.py b/papers/bmodes/scripts/config_space_pte_matrices.py index 474b71a7..3e3f9a67 100644 --- a/papers/bmodes/scripts/config_space_pte_matrices.py +++ b/papers/bmodes/scripts/config_space_pte_matrices.py @@ -789,7 +789,6 @@ def main( pure_eb_pte_files, cosebis_pte_files, output_dir, - spec_path=None, fiducial_overrides=None, ): # Both corrected and uncorrected versions (exclude ecut variants) @@ -964,8 +963,6 @@ def main( # Build evidence evidence_data = { - "spec_id": "config_space_pte_matrices", - "spec_path": spec_path or "papers/bmodes/config/config_space_pte_matrices.md", "generated": datetime.now().isoformat(), "evidence": { "versions": {}, @@ -1016,7 +1013,7 @@ def main( def _versions_config_space(config): - """Reproduce VERSIONS_CONFIG_SPACE_PTES from the Snakemake claims.smk: + """Reproduce VERSIONS_CONFIG_SPACE_PTES from the Snakemake figures.smk: leak-corrected (non-ecut) versions plus their uncorrected counterparts.""" leak_corr = [ v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v @@ -1032,13 +1029,11 @@ def _from_snakemake(smk): else: pure_eb_pte_files = list(pure_eb_pte_files) cosebis_pte_files = list(smk.input["cosebis_pte_files"]) - spec_paths = smk.input["specs"] main( config=smk.config, pure_eb_pte_files=pure_eb_pte_files, cosebis_pte_files=cosebis_pte_files, output_dir=Path(smk.output["evidence"]).parent, - spec_path=spec_paths[0], ) diff --git a/papers/bmodes/scripts/container_env.sh b/papers/bmodes/scripts/container_env.sh deleted file mode 100644 index a63b40f5..00000000 --- a/papers/bmodes/scripts/container_env.sh +++ /dev/null @@ -1,45 +0,0 @@ -# Shared environment for the run_*.sh sweep drivers. Source, don't run: -# -# . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" -# -# Sets the checkout the drivers run out of, the container to run in, and -# `spv_python`, which is how every driver invokes python inside it. -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts - -# CONTAINER and BIND are resolved exactly as `sp_validation/container.py` does: -# the writable sandbox if there is one, else the SIF. -_spv_cache=${XDG_CACHE_HOME:-$HOME/.cache}/sp_validation -_spv_sandbox=${SPV_SANDBOX:-$_spv_cache/sandbox} -if [ -d "$_spv_sandbox" ]; then - CONTAINER=$_spv_sandbox -else - CONTAINER=${SPV_CONTAINER:-$_spv_cache/sp_validation.sif} -fi -BIND=${SPV_APPTAINER_BINDS:-/home,/scratch,/automnt,/n17data,/n23data1,/n09data} -unset _spv_cache _spv_sandbox - -# Every math library pinned to one thread -- pass as SPV_EXEC_EXTRA where the -# parallelism is by process, not by thread. -SINGLE_THREAD_ENV="--env OMP_NUM_THREADS=1 --env OPENBLAS_NUM_THREADS=1 - --env MKL_NUM_THREADS=1 --env NUMBA_NUM_THREADS=1 --env NUMEXPR_NUM_THREADS=1 - --env VECLIB_MAXIMUM_THREADS=1" - -# Run python inside the container against the checkout's src. Extra -# `apptainer exec` flags go in SPV_EXEC_EXTRA (word-split on purpose). -spv_python() { - apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" ${SPV_EXEC_EXTRA:-} \ - "$CONTAINER" /usr/local/bin/python "$@" -} - -# Echo the version list a sweep runs over: $VERSIONS if the caller set one, -# else whatever sweep_versions.py resolves from $1 (a config path). -sweep_versions() { - if [ -n "${VERSIONS:-}" ]; then - echo "$VERSIONS" - else - spv_python "$PSCRIPTS/sweep_versions.py" --config "$1" - fi -} diff --git a/papers/bmodes/scripts/cosebis_binning_comparison.py b/papers/bmodes/scripts/cosebis_binning_comparison.py deleted file mode 100644 index 1e9ac56f..00000000 --- a/papers/bmodes/scripts/cosebis_binning_comparison.py +++ /dev/null @@ -1,320 +0,0 @@ -"""COSEBIS angular binning convergence test. - -Computes COSEBIS B-modes from both 1,000-bin and 10,000-bin ξ± integration -grids and compares. Tests whether the numerical integration is converged: -if B_n values agree, the 1,000-bin results are reliable; if they differ, -integration error may contaminate the anomalous PTE. - -Reference: Asgari et al. 2017 — ≥10,000 bins for E_7 at 0.5% accuracy. -""" - -import argparse -import json -import types -from datetime import datetime -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -import treecorr -from cosmo_numba.B_modes.cosebis import COSEBIS -from plotting_utils import ( - FIG_WIDTH_SINGLE, - PAPER_MPLSTYLE, - compute_chi2_pte, -) - -from sp_validation.b_modes import calculate_cosebis, scale_cut_to_bins - -plt.style.use(PAPER_MPLSTYLE) - - -def _load_gg(xi_path, min_sep, max_sep, nbins, columns_only=False): - """Load a TreeCorr GGCorrelation from a text file. - - If columns_only=True, read only the per-bin columns (meanr, xip, xim) - and skip the patch covariance. This avoids loading a 20000x20000 - covariance matrix for high-nbins files. - """ - gg = treecorr.GGCorrelation( - min_sep=min_sep, - max_sep=max_sep, - nbins=nbins, - sep_units="arcmin", - ) - if columns_only: - # TreeCorr ASCII: ## comment, # col_names, then data rows. - # Read only per-bin columns, skip the huge patch covariance. - # cols: r_nom meanr meanlogr xip xim xip_im xim_im sigma_xip sigma_xim weight npairs - data = np.loadtxt(xi_path, max_rows=nbins) - # Compute bin edges from log-spaced binning (matching TreeCorr) - bin_edges = np.exp(np.linspace(np.log(min_sep), np.log(max_sep), nbins + 1)) - return types.SimpleNamespace( - meanr=data[:, 1], - xip=data[:, 3], - xim=data[:, 4], - left_edges=bin_edges[:-1], - right_edges=bin_edges[1:], - ) - else: - gg.read(xi_path) - return gg - - -def _compute_Bn_only(gg, nmodes, scale_cut): - """Compute COSEBIS B_n from a GGCorrelation without covariance.""" - min_theta, max_theta = scale_cut - start_bin, stop_bin = scale_cut_to_bins(gg, min_theta, max_theta) - inds = np.arange(start_bin, stop_bin) - theta_cut = gg.meanr[inds] - xip_cut = gg.xip[inds] - xim_cut = gg.xim[inds] - - cosebis = COSEBIS( - theta_min=np.min(theta_cut), - theta_max=np.max(theta_cut), - N_max=nmodes, - precision=120, - ) - En, Bn = cosebis.cosebis_from_xipm(theta_cut, xip_cut, xim_cut, parallel=True) - return En, Bn - - -def main(config, xi_1k_path, xi_10k_path, cov_1k_path, out_dir): - nmodes = config["fiducial"]["nmodes"] - - # Scale cuts - fiducial_scale_cut = ( - float(config["fiducial"]["fiducial_min_scale"]), - float(config["fiducial"]["fiducial_max_scale"]), - ) - full_scale_cut = ( - float(config["cosebis"]["theta_min"]), - float(config["cosebis"]["theta_max"]), - ) - scale_cuts = {"fiducial": fiducial_scale_cut, "full": full_scale_cut} - - # Integration parameters - min_sep_int = float(config["fiducial"]["min_sep_int"]) - max_sep_int = float(config["fiducial"]["max_sep_int"]) - nbins_1k = int(config["fiducial"]["nbins_int"]) - nbins_10k = 10_000 - - # Load both ξ± grids - print("Loading 1,000-bin ξ±...") - gg_1k = _load_gg(xi_1k_path, min_sep_int, max_sep_int, nbins_1k) - print("Loading 10,000-bin ξ±...") - gg_10k = _load_gg( - xi_10k_path, min_sep_int, max_sep_int, nbins_10k, columns_only=True - ) - - # Compute COSEBIS from 1,000-bin ξ± (with covariance for PTE baseline) - print("\nComputing COSEBIS from 1,000-bin ξ±...") - results_1k = calculate_cosebis( - gg_1k, - nmodes=nmodes, - scale_cuts=list(scale_cuts.values()), - cov_path=cov_1k_path, - ) - - # Compute COSEBIS from 10,000-bin ξ± (B_n only — no matching covariance) - print("\nComputing COSEBIS from 10,000-bin ξ±...") - results_10k = {} - for scale_key, scale_cut in scale_cuts.items(): - En, Bn = _compute_Bn_only(gg_10k, nmodes, scale_cut) - results_10k[scale_key] = {"En": En, "Bn": Bn} - print(f" {scale_key} {scale_cut}: done") - - # Compare and compute PTEs using 1k covariance for both - output_dir = Path(out_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - evidence = {} - for scale_key, scale_cut in scale_cuts.items(): - r1k = results_1k[scale_cut] - r10k = results_10k[scale_key] - - Bn_1k = r1k["Bn"] - Bn_10k = r10k["Bn"] - cov_B = r1k["cov"][nmodes:, nmodes:] - sigma_B = np.sqrt(np.diag(cov_B)) - - # Fractional difference - delta_Bn = Bn_10k - Bn_1k - delta_Bn_sigma = delta_Bn / sigma_B - - # PTEs using the same covariance - chi2_1k, pte_1k, dof = compute_chi2_pte(Bn_1k, cov_B) - chi2_10k, pte_10k, _ = compute_chi2_pte(Bn_10k, cov_B) - - # Also for first 6 modes - cov_B_6 = cov_B[:6, :6] - chi2_1k_6, pte_1k_6, _ = compute_chi2_pte(Bn_1k[:6], cov_B_6) - chi2_10k_6, pte_10k_6, _ = compute_chi2_pte(Bn_10k[:6], cov_B_6) - - prefix = scale_key - evidence[f"{prefix}_max_delta_Bn_over_sigma"] = float( - np.max(np.abs(delta_Bn_sigma)) - ) - evidence[f"{prefix}_rms_delta_Bn_over_sigma"] = float( - np.sqrt(np.mean(delta_Bn_sigma**2)) - ) - evidence[f"{prefix}_pte_1k_20"] = float(pte_1k) - evidence[f"{prefix}_pte_10k_20"] = float(pte_10k) - evidence[f"{prefix}_chi2_1k_20"] = float(chi2_1k) - evidence[f"{prefix}_chi2_10k_20"] = float(chi2_10k) - evidence[f"{prefix}_pte_1k_6"] = float(pte_1k_6) - evidence[f"{prefix}_pte_10k_6"] = float(pte_10k_6) - evidence[f"{prefix}_chi2_1k_6"] = float(chi2_1k_6) - evidence[f"{prefix}_chi2_10k_6"] = float(chi2_10k_6) - - print(f"\n{scale_key} [{scale_cut[0]:.0f}–{scale_cut[1]:.0f}']:") - print(f" max |ΔB_n/σ| = {np.max(np.abs(delta_Bn_sigma)):.4f}") - print(f" RMS ΔB_n/σ = {np.sqrt(np.mean(delta_Bn_sigma**2)):.4f}") - print(f" PTE (20 modes): 1k={pte_1k:.4e} 10k={pte_10k:.4e}") - print(f" PTE ( 6 modes): 1k={pte_1k_6:.4e} 10k={pte_10k_6:.4e}") - - # Figure: side-by-side comparison - fig, axes = plt.subplots( - 1, 2, figsize=(FIG_WIDTH_SINGLE * 2, FIG_WIDTH_SINGLE * 0.6), sharey=True - ) - - colors = sns.color_palette("colorblind", 4) - modes = np.arange(1, nmodes + 1) - - for ax, (scale_key, scale_cut) in zip(axes, scale_cuts.items()): - r1k = results_1k[scale_cut] - r10k = results_10k[scale_key] - - cov_B = r1k["cov"][nmodes:, nmodes:] - sigma_B = np.sqrt(np.diag(cov_B)) - - Bn_1k_norm = r1k["Bn"] / sigma_B - Bn_10k_norm = r10k["Bn"] / sigma_B - - ax.errorbar( - modes - 0.15, - Bn_1k_norm, - yerr=np.ones(nmodes), - fmt="o", - color=colors[0], - markerfacecolor=colors[0], - markeredgecolor="white", - markeredgewidth=0.5, - markersize=5, - capsize=2, - capthick=0.8, - linewidth=0.8, - elinewidth=0.8, - label=r"1,000 bins", - ) - ax.errorbar( - modes + 0.15, - Bn_10k_norm, - yerr=np.ones(nmodes), - fmt="s", - color=colors[1], - markerfacecolor=colors[1], - markeredgecolor="white", - markeredgewidth=0.5, - markersize=5, - capsize=2, - capthick=0.8, - linewidth=0.8, - elinewidth=0.8, - label=r"10,000 bins", - ) - - ax.axhline(0, color="black", linewidth=0.8, alpha=0.6) - ax.axvspan(0.5, 6.5, color="0.95", alpha=0.5, zorder=0) - - # Annotate PTEs - pte_1k = evidence[f"{scale_key}_pte_1k_20"] - pte_10k = evidence[f"{scale_key}_pte_10k_20"] - ax.text( - 0.97, - 0.97, - f"PTE(20): {pte_1k:.2e} → {pte_10k:.2e}", - transform=ax.transAxes, - ha="right", - va="top", - fontsize=7, - bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.8), - ) - - ax.set_xlabel("COSEBIS mode $n$") - ax.set_xlim(0.5, nmodes + 0.5) - ax.set_xticks(np.arange(1, nmodes + 1)) - ax.set_title( - rf"$\theta = {scale_cut[0]:.0f}$--${scale_cut[1]:.0f}'$", fontsize=10 - ) - - axes[0].set_ylabel(r"$B_n / \sigma_n$") - axes[0].legend(loc="upper left", frameon=True, framealpha=0.9, fontsize=8) - - plt.tight_layout() - fig_path = output_dir / "figure.png" - fig.savefig(fig_path, dpi=300, bbox_inches="tight") - print(f"\nSaved {fig_path}") - plt.close(fig) - - # Write evidence - evidence_data = { - "id": "cosebis_binning_comparison", - "generated": datetime.now().isoformat(), - "input": { - "xi_1k": str(xi_1k_path), - "xi_10k": str(xi_10k_path), - "cov_1k": str(cov_1k_path), - }, - "output": {"figure": "figure.png"}, - "params": { - "nbins_1k": nbins_1k, - "nbins_10k": nbins_10k, - "nmodes": nmodes, - "fiducial_scale_cut": list(fiducial_scale_cut), - "full_scale_cut": list(full_scale_cut), - }, - "evidence": evidence, - } - - evidence_path = output_dir / "evidence.json" - with open(evidence_path, "w") as f: - json.dump(evidence_data, f, indent=2) - print(f"Saved evidence to {evidence_path}") - - -def _from_cli(argv=None): - import yaml - - ap = argparse.ArgumentParser( - description="COSEBI angular-binning convergence (1,000 vs 10,000-bin xi_pm) figure." - ) - ap.add_argument( - "--config", required=True, help="Absolute path to bmodes config.yaml" - ) - ap.add_argument( - "--xi-1k", - required=True, - help="Fiducial 1000-bin integration-grid TreeCorr xi_pm .txt", - ) - ap.add_argument( - "--xi-10k", - required=True, - help="Fiducial 10000-bin integration-grid TreeCorr xi_pm .txt", - ) - ap.add_argument( - "--cov-1k", - required=True, - help="Fiducial 1000-bin Gaussian covariance (processed .txt) used for both grids", - ) - ap.add_argument("--out", required=True, help="Output directory (lc {output})") - a = ap.parse_args(argv) - with open(a.config) as f: - config = yaml.safe_load(f) - main(config, a.xi_1k, a.xi_10k, a.cov_1k, a.out) - - -if __name__ == "__main__": - _from_cli() diff --git a/papers/bmodes/scripts/cosebis_data_vector.py b/papers/bmodes/scripts/cosebis_data_vector.py index dfa0383b..89631f83 100644 --- a/papers/bmodes/scripts/cosebis_data_vector.py +++ b/papers/bmodes/scripts/cosebis_data_vector.py @@ -1,4 +1,4 @@ -"""COSEBIs data vector claim. +"""COSEBIs data vector figure. Single-panel figure showing B-mode COSEBIS for each catalog version. Overplots fiducial and full angular range scale cuts. @@ -207,7 +207,6 @@ def main(config, xi_integration, cov_integration, out_dir): plt.close(fig) evidence_data = { - "spec_id": "cosebis_data_vector", "generated": datetime.now().isoformat(), "evidence": { "version": version, diff --git a/papers/bmodes/scripts/cosebis_version_comparison.py b/papers/bmodes/scripts/cosebis_version_comparison.py index 9942bb28..65543ac9 100644 --- a/papers/bmodes/scripts/cosebis_version_comparison.py +++ b/papers/bmodes/scripts/cosebis_version_comparison.py @@ -1,4 +1,4 @@ -"""COSEBIS version comparison claim. +"""COSEBIS version comparison figure. Visualizes B-mode COSEBIS across catalog versions. Produces figures at fiducial scale cut and full range. @@ -165,7 +165,7 @@ def main( fiducial_xi_path=None, fiducial_cov_path=None, ): - # Leak-corrected, non-ecut versions (matches VERSIONS_LEAK_CORR in claims.smk) + # Leak-corrected versions (matches VERSIONS_LEAK_CORR in figures.smk) versions = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] nmodes = config["fiducial"]["nmodes"] plotting_config = config["plotting"] @@ -342,7 +342,6 @@ def main( evidence_versions[f"{version}_{key}"] = val evidence_data = { - "spec_id": "cosebis_version_comparison", "generated": datetime.now().isoformat(), "evidence": { "scale_cuts": scale_cuts, diff --git a/papers/bmodes/scripts/covariance_blind_consistency.py b/papers/bmodes/scripts/covariance_blind_consistency.py deleted file mode 100644 index 99ff920c..00000000 --- a/papers/bmodes/scripts/covariance_blind_consistency.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Covariance Blind Consistency Claim - -Compare reporting covariance diagonals across blinds A, B, C. -Claim: diagonals agree at percent level. -""" - -import json -from datetime import datetime -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns - -plt.style.use( - "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/notebooks/" - "2D_cosmic_shear_paper_plots/config/paper.mplstyle" -) - - -def load_covariance_diagonal(path, nbins=20): - """Load covariance and extract ξ+ and ξ- diagonals.""" - cov = np.loadtxt(path) - # Covariance is 2*nbins x 2*nbins: [ξ+, ξ-] - diag = np.diag(cov) - return { - "xip": diag[:nbins], - "xim": diag[nbins:], - } - - -def compute_ratios(diag_ref, diag_test): - """Compute ratio and deviation statistics.""" - ratios = {} - for key in ["xip", "xim"]: - ratio = diag_test[key] / diag_ref[key] - dev = np.abs(ratio - 1.0) - ratios[key] = { - "ratio": ratio, - "max_dev": float(np.max(dev)), - "mean_dev": float(np.mean(dev)), - } - return ratios - - -def make_figure(theta, ratios_B, ratios_C, output_path): - """Two-panel ratio plot with threshold bands.""" - fig, axes = plt.subplots(1, 2, figsize=(10, 4), sharey=True) - - colors = sns.color_palette("colorblind", 2) - - for ax, key, label in zip(axes, ["xip", "xim"], [r"$\xi_+$", r"$\xi_-$"]): - ax.axhline(1.0, color="gray", ls="-", lw=0.8, zorder=0) - ax.axhspan(0.99, 1.01, color="teal", alpha=0.15, label=r"$\pm 1\%$") - ax.axhspan(0.90, 1.10, color="orange", alpha=0.10, label=r"$\pm 10\%$") - - ax.plot( - theta, - ratios_B[key]["ratio"], - "o-", - color=colors[0], - label="B / A", - markersize=5, - ) - ax.plot( - theta, - ratios_C[key]["ratio"], - "s--", - color=colors[1], - label="C / A", - markersize=5, - ) - - ax.set_xscale("log") - ax.set_xlabel(r"$\theta$ [arcmin]") - ax.set_title(label) - - axes[0].set_ylabel("Diagonal ratio") - axes[0].legend(loc="upper right", fontsize=8) - - fig.tight_layout() - fig.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def main(snakemake): - # Load covariances (lowercase names match rule definition) - diag_A = load_covariance_diagonal(snakemake.input.cov_a) - diag_B = load_covariance_diagonal(snakemake.input.cov_b) - diag_C = load_covariance_diagonal(snakemake.input.cov_c) - - # Compute ratios - ratios_B = compute_ratios(diag_A, diag_B) - ratios_C = compute_ratios(diag_A, diag_C) - - # Angular bins (log-spaced) - min_sep = snakemake.config["fiducial"]["min_sep"] - max_sep = snakemake.config["fiducial"]["max_sep"] - nbins = snakemake.config["fiducial"]["nbins"] - theta = np.logspace(np.log10(min_sep), np.log10(max_sep), nbins) - - # Generate figure - make_figure(theta, ratios_B, ratios_C, snakemake.output.figure) - - # Determine pass/fail - all_max_devs = [ - ratios_B["xip"]["max_dev"], - ratios_B["xim"]["max_dev"], - ratios_C["xip"]["max_dev"], - ratios_C["xim"]["max_dev"], - ] - pass_1pct = all(d < 0.01 for d in all_max_devs) - pass_10pct = all(d < 0.10 for d in all_max_devs) - - # Build evidence - evidence = { - "spec_id": "covariance_blind_consistency", - "spec_path": "workflow/config/covariance_blind_consistency.md", - "depends_on": ["covariance"], - "generated": datetime.now().isoformat(), - "evidence": { - "xip": { - "B_to_A": { - "max_dev": ratios_B["xip"]["max_dev"], - "mean_dev": ratios_B["xip"]["mean_dev"], - }, - "C_to_A": { - "max_dev": ratios_C["xip"]["max_dev"], - "mean_dev": ratios_C["xip"]["mean_dev"], - }, - }, - "xim": { - "B_to_A": { - "max_dev": ratios_B["xim"]["max_dev"], - "mean_dev": ratios_B["xim"]["mean_dev"], - }, - "C_to_A": { - "max_dev": ratios_C["xim"]["max_dev"], - "mean_dev": ratios_C["xim"]["mean_dev"], - }, - }, - "pass_1pct": pass_1pct, - "pass_10pct": pass_10pct, - }, - "output": { - "figure": Path(snakemake.output.figure).name, - }, - } - - # Write evidence - Path(snakemake.output.evidence).parent.mkdir(parents=True, exist_ok=True) - with open(snakemake.output.evidence, "w") as f: - json.dump(evidence, f, indent=2) - - -if __name__ == "__main__": - main(snakemake) # noqa: F821 diff --git a/papers/bmodes/scripts/explorations/compare_mask_effect.py b/papers/bmodes/scripts/explorations/compare_mask_effect.py deleted file mode 100644 index 0c5f8d6f..00000000 --- a/papers/bmodes/scripts/explorations/compare_mask_effect.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -# %% -""" -Visual comparison of CosmoCov covariance matrices with and without mask window. - -This script loads processed covariance matrices from the Snakemake exploration -and produces diagnostic plots highlighting the impact of applying the CosmoCov -mask power spectrum file. -""" - -# %% -from pathlib import Path -from typing import Tuple - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from IPython import get_ipython - -# %% -ipython = get_ipython() -if ipython is not None: - ipython.run_line_magic("matplotlib", "inline") - -plt.style.use( - "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/" - "notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle" -) - - -# %% -def load_covariance_pair( - mask_path: Path, nomask_path: Path -) -> Tuple[np.ndarray, np.ndarray]: - """Load covariance matrices for mask/no-mask comparison.""" - if not mask_path.exists(): - raise FileNotFoundError(f"Mask covariance not found: {mask_path}") - if not nomask_path.exists(): - raise FileNotFoundError(f"No-mask covariance not found: {nomask_path}") - - mask_cov = np.loadtxt(mask_path) - nomask_cov = np.loadtxt(nomask_path) - - if mask_cov.shape != nomask_cov.shape: - raise ValueError( - f"Covariance shapes differ: mask {mask_cov.shape}, no-mask {nomask_cov.shape}" - ) - - return mask_cov, nomask_cov - - -# %% -def create_diagnostics( - mask_cov: np.ndarray, nomask_cov: np.ndarray -) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - """Compute standard deviations, variance ratios, and covariance differences.""" - mask_std = np.sqrt(np.diag(mask_cov)) - nomask_std = np.sqrt(np.diag(nomask_cov)) - - # Protect against divide-by-zero in variance ratio - with np.errstate(divide="ignore", invalid="ignore"): - variance_ratio = np.square(mask_std) / np.square(nomask_std) - variance_ratio[~np.isfinite(variance_ratio)] = np.nan - - covariance_delta = mask_cov - nomask_cov - - return mask_std, nomask_std, variance_ratio, covariance_delta - - -# %% -def plot_comparison( - mask_std: np.ndarray, - nomask_std: np.ndarray, - variance_ratio: np.ndarray, - covariance_delta: np.ndarray, - output_path: Path, -) -> None: - """Generate diagnostic plots comparing masked vs nominal covariance.""" - sns.set_palette("husl", 2) - - n_points = mask_std.size - indices = np.arange(n_points) - - fig = plt.figure(figsize=(12, 10)) - grid = fig.add_gridspec(2, 2, height_ratios=[1.0, 1.2]) - - ax_std = fig.add_subplot(grid[0, 0]) - ax_ratio = fig.add_subplot(grid[0, 1]) - ax_delta = fig.add_subplot(grid[1, :]) - - ax_std.plot(indices, nomask_std, label="No mask", linewidth=2) - ax_std.plot(indices, mask_std, label="Mask applied", linewidth=2) - ax_std.set_xlabel("Data vector index") - ax_std.set_ylabel("Standard deviation") - ax_std.set_title("Marginal uncertainties") - ax_std.grid(alpha=0.3) - ax_std.legend() - - ax_ratio.plot(indices, variance_ratio, linewidth=2, color="#ef8a62") - ax_ratio.set_xlabel("Data vector index") - ax_ratio.set_ylabel( - "$\\sigma^2_{\\mathrm{mask}} / \\sigma^2_{\\mathrm{no\\ mask}}$" - ) - ax_ratio.set_title("Variance ratio") - ax_ratio.grid(alpha=0.3) - - im = ax_delta.imshow( - covariance_delta, - cmap="PRGn", - vmin=-np.nanmax(np.abs(covariance_delta)), - vmax=np.nanmax(np.abs(covariance_delta)), - origin="upper", - ) - ax_delta.set_title("Covariance difference (mask - no mask)") - ax_delta.set_xlabel("Data vector index") - ax_delta.set_ylabel("Data vector index") - fig.colorbar(im, ax=ax_delta, fraction=0.046, pad=0.04, label="$\\Delta C$") - - variance_reduction = np.nanmedian(variance_ratio) - ax_ratio.text( - 0.02, - 0.95, - f"Median variance ratio: {variance_reduction:.3f}", - transform=ax_ratio.transAxes, - ha="left", - va="top", - fontsize=11, - bbox=dict(boxstyle="round", facecolor="white", alpha=0.6), - ) - - fig.tight_layout() - fig.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -# %% -def main() -> None: - try: - mask_path = Path(snakemake.input["mask"]) # type: ignore[name-defined] - nomask_path = Path(snakemake.input["nomask"]) # type: ignore[name-defined] - output_path = Path(snakemake.output["plot"]) # type: ignore[name-defined] - except NameError as exc: # pragma: no cover - executed only outside Snakemake - raise RuntimeError( - "This script is intended to be executed via Snakemake." - ) from exc - - mask_cov, nomask_cov = load_covariance_pair(mask_path, nomask_path) - mask_std, nomask_std, variance_ratio, covariance_delta = create_diagnostics( - mask_cov, nomask_cov - ) - - output_path.parent.mkdir(parents=True, exist_ok=True) - plot_comparison(mask_std, nomask_std, variance_ratio, covariance_delta, output_path) - - -# %% -if __name__ == "__main__": - main() diff --git a/papers/bmodes/scripts/explorations/plot_mask_diagonal_ratio.py b/papers/bmodes/scripts/explorations/plot_mask_diagonal_ratio.py deleted file mode 100644 index 32f930b5..00000000 --- a/papers/bmodes/scripts/explorations/plot_mask_diagonal_ratio.py +++ /dev/null @@ -1,89 +0,0 @@ -import argparse -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns - -STYLE_PATH = Path( - "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/config/paper.mplstyle" -) - - -def compute_ratio( - mask_path: Path, nomask_path: Path, data_out: Path, plot_out: Path -) -> None: - mask_cov = np.loadtxt(mask_path) - nomask_cov = np.loadtxt(nomask_path) - - if mask_cov.shape != nomask_cov.shape: - raise ValueError( - f"Mask and no-mask covariances have mismatched shapes {mask_cov.shape} vs {nomask_cov.shape}" - ) - - nomask_diag = np.diag(nomask_cov) - mask_diag = np.diag(mask_cov) - - if np.any(nomask_diag == 0): - zero_indices = np.where(nomask_diag == 0)[0] - raise ValueError( - f"Zero variance entries at indices {zero_indices.tolist()} prevent ratio evaluation." - ) - - ratio = mask_diag / nomask_diag - - data_out.parent.mkdir(parents=True, exist_ok=True) - np.savetxt(data_out, ratio) - - plt.style.use(STYLE_PATH) - sns.set_palette("husl", 1) - - fig, ax = plt.subplots(figsize=(9, 4.5), constrained_layout=True) - x = np.arange(ratio.size) - - ax.plot(x, ratio, linewidth=1.5) - ax.axhline(1.0, color="black", linestyle="--", linewidth=1.0) - ax.set_xlabel("Covariance diagonal element") - ax.set_ylabel(r"$C_{\mathrm{mask}} / C_{\mathrm{nomask}}$") - ax.set_title("Gaussian covariance diagonal ratio") - ax.set_xlim(0, ratio.size - 1) - - plot_out.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(plot_out, dpi=300) - - -def run_from_snakemake() -> None: - mask_path = Path(snakemake.input["mask"]) - nomask_path = Path(snakemake.input["nomask"]) - data_out = Path(snakemake.output["data"]) - plot_out = Path(snakemake.output["plot"]) - compute_ratio(mask_path, nomask_path, data_out, plot_out) - - -def run_from_cli() -> None: - parser = argparse.ArgumentParser( - description="Plot ratio of covariance diagonals with and without mask." - ) - parser.add_argument( - "mask", - type=Path, - help="Path to Gaussian covariance matrix generated with mask.", - ) - parser.add_argument( - "nomask", - type=Path, - help="Path to Gaussian covariance matrix generated without mask.", - ) - parser.add_argument( - "data", type=Path, help="Output path for diagonal ratio text file." - ) - parser.add_argument("plot", type=Path, help="Output path for diagonal ratio plot.") - args = parser.parse_args() - - compute_ratio(args.mask, args.nomask, args.data, args.plot) - - -if "snakemake" in globals(): - run_from_snakemake() -else: - run_from_cli() diff --git a/papers/bmodes/scripts/filter_catalog_ellipticity.py b/papers/bmodes/scripts/filter_catalog_ellipticity.py deleted file mode 100644 index baab3ffb..00000000 --- a/papers/bmodes/scripts/filter_catalog_ellipticity.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Filter a shear catalog by ellipticity magnitude. - -Applies |e| < e_max using leak-corrected ellipticity columns, -writes filtered FITS, and computes survey properties (n_eff, sigma_e) -using the parent version's area. -""" - -import json -import sys - -import numpy as np -from astropy.io import fits - -sys.stdout = ( - sys.stdout if hasattr(sys, "ps1") else open(sys.stdout.fileno(), "w", buffering=1) -) -sys.stderr = ( - sys.stderr if hasattr(sys, "ps1") else open(sys.stderr.fileno(), "w", buffering=1) -) - - -input_path = snakemake.input["catalog"] -output_fits = snakemake.output["catalog"] -output_props = snakemake.output["survey_props"] - -e_max = float(snakemake.params["e_max"]) -e1_col = snakemake.params["e1_col"] -e2_col = snakemake.params["e2_col"] -w_col = snakemake.params["w_col"] -parent_area_deg2 = float(snakemake.params["parent_area_deg2"]) - -print(f"Loading catalog: {input_path}") -with fits.open(input_path, memmap=True) as hdul: - data = hdul[1].data - n_total = len(data) - - e1 = np.asarray(data[e1_col], dtype=np.float32) - e2 = np.asarray(data[e2_col], dtype=np.float32) - e_mag = np.sqrt(e1**2 + e2**2) - del e1, e2 - - mask = e_mag < e_max - del e_mag - n_kept = int(np.sum(mask)) - print( - f"Ellipticity cut |e| < {e_max}: {n_kept}/{n_total} galaxies kept ({100 * n_kept / n_total:.1f}%)" - ) - - # Compute survey properties from masked columns (memory-efficient) - w = np.asarray(data[w_col][mask], dtype=np.float64) - e1_filt = np.asarray(data[e1_col][mask], dtype=np.float64) - e2_filt = np.asarray(data[e2_col][mask], dtype=np.float64) - - sum_w = float(np.sum(w)) - sum_w2 = float(np.sum(w**2)) - sum_w2_e2 = float(np.sum(w**2 * (e1_filt**2 + e2_filt**2))) - del w, e1_filt, e2_filt - - filtered = data[mask] - del mask - - print(f"Writing filtered catalog: {output_fits}") - fits.writeto(output_fits, filtered, overwrite=True) - del filtered - -area_arcmin2 = parent_area_deg2 * 3600.0 -n_eff = sum_w**2 / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 -sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 - -props = { - "parent_area_deg2": parent_area_deg2, - "n_eff": n_eff, - "sigma_e": sigma_e, - "n_total": n_total, - "n_kept": n_kept, - "fraction_kept": n_kept / n_total, - "e_max": e_max, -} - -print(f"Survey properties: n_eff={n_eff:.3f} gal/arcmin², sigma_e={sigma_e:.4f}") -print(" (parent n_eff and sigma_e should be compared to assess impact)") - -with open(output_props, "w") as f: - json.dump(props, f, indent=2) - -print("Done.") diff --git a/papers/bmodes/scripts/generate_paper_macros.py b/papers/bmodes/scripts/generate_paper_macros.py index 35442b03..9e02c132 100644 --- a/papers/bmodes/scripts/generate_paper_macros.py +++ b/papers/bmodes/scripts/generate_paper_macros.py @@ -1,10 +1,10 @@ -"""Generate LaTeX macros from claim evidence. +"""Generate LaTeX macros and PTE tables from the figure rules' evidence.json. -Reads evidence.json files and produces: +Produces: - claims_macros.tex: LaTeX macro definitions for paper values - pte_table_results.tex: PTE results table for main text - pte_table_appendix.tex: PTE table for appendix -- evidence.json: Dashboard dependency tracking +- evidence.json (CLI only): which inputs the tables were built from """ import json @@ -78,17 +78,16 @@ def generate_macros( ): """Generate LaTeX macros from evidence files. - Macro names are kept simple. The spec (bmodes_paper.md) - determines which values go into the paper. Fiducial version from config. + Macro names are kept simple. Fiducial version from config. ``config_pte_path`` / ``harmonic_pte_path`` override the - ``claims_dir//evidence.json`` location for the two PTE-matrix + ``claims_dir//evidence.json`` location for the two PTE-matrix evidence files (used by the CLI form, where each lc output lands at its own absolute results path rather than under a shared tapestry tree). When - ``None`` the original ``claims_dir``-relative layout is used. + ``None`` the ``claims_dir``-relative layout is used. """ macros = [] - macros.append("% Auto-generated from claim evidence") + macros.append("% Auto-generated from figure evidence.json summaries") macros.append("% Regenerate: snakemake paper_macros") macros.append("% See workflow/config/bmodes_paper.md for paper choices") macros.append("") @@ -138,7 +137,7 @@ def generate_macros( with open(eb_path) as f: eb_ev = json.load(f).get("evidence", {}) - macros.append("% pure_eb_data_vector (min across blinds per spec)") + macros.append("% pure_eb_data_vector (min across blinds)") # Fiducial PTEs - use pte_joint_min (conservative across blinds) eb_fid = eb_ev.get("fiducial", {}) @@ -592,14 +591,11 @@ def generate_pte_tables( def generate_evidence( - spec_id: str, - spec_path: str, depends_on: list[str], claims_dir: Path, output_path: Path, ): - """Generate evidence.json for dashboard dependency tracking.""" - # Collect summary from dependent claims + """Record which evidence files the tables were built from.""" summary = {} for dep in depends_on: dep_evidence = claims_dir / dep / "evidence.json" @@ -614,8 +610,6 @@ def generate_evidence( summary[dep] = {"has_evidence": False} evidence = { - "spec_id": spec_id, - "spec_path": spec_path, "depends_on": depends_on, "generated": datetime.now().isoformat(), "evidence": { @@ -640,40 +634,19 @@ def _from_snakemake(smk): versions = config["versions"] version_labels = config["plotting"]["version_labels"] - # Separate macro file from PTE tables and evidence - # Only claims_macros.tex gets macro content; PTE tables generated separately macro_file = [Path(p) for p in smk.output if p.endswith("claims_macros.tex")] - evidence_outputs = [Path(p) for p in smk.output if p.endswith("evidence.json")] print(f"Generating macros from {tapestry_dir}") generate_macros(tapestry_dir, macro_file, fiducial_version) - # Generate PTE tables (separate files, not macro content) - if macro_file: + # The B-modes paper rule also writes the two PTE tables beside its macros + if len(smk.output) > 1: paper_dir = macro_file[0].parent print(f"Generating PTE tables to {paper_dir}") generate_pte_tables( tapestry_dir, paper_dir, fiducial_version, versions, version_labels, config ) - # Generate evidence.json if requested - # Dependencies derived from snakemake inputs (rules.X.output declarations) - rule_inputs = smk.input.keys() - input_deps = [ - k for k in rule_inputs if k.endswith("_evidence") or k == "covariance_evidence" - ] - depends_on = [d.replace("_evidence", "") for d in input_deps] - - for evidence_path in evidence_outputs: - spec_id = evidence_path.parent.name # e.g., xi_cosmology_paper - generate_evidence( - spec_id=spec_id, - spec_path=f"workflow/config/{spec_id}.md", - depends_on=depends_on, - claims_dir=tapestry_dir, - output_path=evidence_path, - ) - def _from_cli(argv=None): import argparse @@ -701,7 +674,7 @@ def _from_cli(argv=None): "--claims-dir", default=None, help=( - "Optional tapestry-style dir holding /evidence.json for the " + "Optional tapestry-style dir holding /evidence.json for the " "extra claims_macros.tex macros (cosebis/pure_eb/harmonic_config); " "the two PTE tables need only the two --*-evidence paths above." ), @@ -746,8 +719,6 @@ def _from_cli(argv=None): ) generate_evidence( - spec_id="pte_summary_evidence", - spec_path="analyses/null_tests/astra.yaml#pte_summary_evidence", depends_on=["config_space_pte_matrices", "harmonic_space_pte_matrices"], claims_dir=claims_dir, output_path=out_dir / "evidence.json", diff --git a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py index dd34db22..b3b05c27 100644 --- a/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py +++ b/papers/bmodes/scripts/harmonic_config_cosebis_comparison.py @@ -510,12 +510,12 @@ def _make_version_comparison_figure( return fig -def main(config, inputs, scale_cut, output_dir, paper_figure_name=None, spec_path=None): +def main(config, inputs, scale_cut, output_dir, paper_figure_name=None): """Harmonic-vs-config COSEBI cross-check for one angular range. ``inputs`` is a dict mirroring ``snakemake.input``: per-version keys ``pseudo_cl_{ver}`` / ``pseudo_cl_cov_{ver}`` / ``xi_{ver}`` / ``cov_{ver}`` - (all absolute paths) plus ``specs``. All artifacts land under ``output_dir``; + (all absolute paths). All artifacts land under ``output_dir``; ``paper_figure_name`` (when set) is the combined 2×2 paper PDF filename. """ nmodes = int(config["fiducial"]["nmodes"]) @@ -765,9 +765,6 @@ def main(config, inputs, scale_cut, output_dir, paper_figure_name=None, spec_pat # --- Evidence --- evidence = { - "spec_id": "harmonic_config_cosebis_comparison", - "spec_path": spec_path - or "papers/bmodes/config/harmonic_config_cosebis_comparison.md", "generated": datetime.now().isoformat(), "evidence": { "nmodes": nmodes, @@ -818,7 +815,7 @@ def _cov_integration_path(cov_dir, version, blind, min_sep, max_sep, nbins): def _angular_ranges(config): - """Reproduce claims.smk _COSEBIS_ANGULAR_RANGES.""" + """Reproduce figures.smk _COSEBIS_ANGULAR_RANGES.""" return { "full": ( float(config["cosebis"]["theta_min"]), @@ -839,7 +836,6 @@ def _from_snakemake(smk): if "paper_figure" in smk.output.keys() else None ), - spec_path=smk.input["specs"][0], ) @@ -929,7 +925,7 @@ def _from_cli(argv=None): npatch = fid["npatch"] versions = _versions_all_for_plots(config) - inputs = {"specs": ["papers/bmodes/config/harmonic_config_cosebis_comparison.md"]} + inputs = {} for ver in versions: is_fiducial = ver == a.fiducial_version @@ -974,7 +970,6 @@ def _from_cli(argv=None): scale_cut=scale_cut, output_dir=a.out, paper_figure_name=f"harmonic_config_cosebis_{a.angular_range}.pdf", - spec_path=inputs["specs"][0], ) diff --git a/papers/bmodes/scripts/harmonic_space_pte_matrices.py b/papers/bmodes/scripts/harmonic_space_pte_matrices.py index aeb031c6..3be9644a 100644 --- a/papers/bmodes/scripts/harmonic_space_pte_matrices.py +++ b/papers/bmodes/scripts/harmonic_space_pte_matrices.py @@ -366,7 +366,7 @@ def main( output_dir.mkdir(parents=True, exist_ok=True) # Per-version pseudo-Cl / covariance files (canonical COSMO_VAL tree, blind A). - # Mirrors _pseudo_cl_path(ver) / _pseudo_cl_cov_path(ver, blind) in claims.smk. + # Mirrors _pseudo_cl_path(ver) / _pseudo_cl_cov_path(ver, blind) in figures.smk. # Fiducial-provenance repoint: when --fiducial-version matches and an explicit # lc override path is set, read that path instead of the reconstructed pattern # (lc files lack the blind=/powspace_nbins= tokens, distinguished by directory). @@ -491,7 +491,6 @@ def main( # Build evidence evidence_data = { - "spec_id": "harmonic_space_pte_matrices", "generated": datetime.now().isoformat(), "evidence": { "blind": fiducial_blind, diff --git a/papers/bmodes/scripts/plot_cosebis_filter_overlay.py b/papers/bmodes/scripts/plot_cosebis_filter_overlay.py index 68b255ab..5f071bb6 100644 --- a/papers/bmodes/scripts/plot_cosebis_filter_overlay.py +++ b/papers/bmodes/scripts/plot_cosebis_filter_overlay.py @@ -183,13 +183,7 @@ def main(): make_figure(ell_32, bb, sigma_bb, ell_dense, Wn_full, Wn_fid, output_figure) # --- Evidence --- - spec_paths = snakemake.input.specs # noqa: F821 - depends_on = [Path(p).stem for p in spec_paths[1:]] - evidence = { - "id": Path(spec_paths[0]).stem, - "kind": "claim", - "depends_on": depends_on, "generated": datetime.now().isoformat(), "evidence": { "nmodes_shown": nmodes, diff --git a/papers/bmodes/scripts/plot_presentation_blind_nz.py b/papers/bmodes/scripts/plot_presentation_blind_nz.py deleted file mode 100644 index c71fbd4f..00000000 --- a/papers/bmodes/scripts/plot_presentation_blind_nz.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Plot the three blinded n(z) curves used in the Moriond slides.""" - -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from plotting_utils import PAPER_MPLSTYLE - - -def load_nz(path): - data = np.loadtxt(path) - return data[:, 0], data[:, 1] - - -plt.style.use(PAPER_MPLSTYLE) -sns.set_palette("husl", 3) - -fig, ax = plt.subplots(figsize=(4.5, 3.2)) - -for label, path in [ - ("Blind A", snakemake.input.nz_A), - ("Blind B", snakemake.input.nz_B), - ("Blind C", snakemake.input.nz_C), -]: - z, nz = load_nz(path) - ax.plot(z, nz, linewidth=1.6, label=label) - -ax.set_xlabel(r"$z$", fontsize=18) -ax.set_ylabel(r"$n(z)$", fontsize=18) -ax.set_xlim(0, 2) -ax.set_ylim(bottom=0) -ax.legend(frameon=False, loc="upper right", fontsize=15) -ax.tick_params(labelsize=15) -ax.grid(False) - -fig.tight_layout() -Path(snakemake.output[0]).parent.mkdir(parents=True, exist_ok=True) -fig.savefig(snakemake.output[0], dpi=220, bbox_inches="tight") diff --git a/papers/bmodes/scripts/plot_pure_eb_covariance.py b/papers/bmodes/scripts/plot_pure_eb_covariance.py deleted file mode 100644 index 5522ef59..00000000 --- a/papers/bmodes/scripts/plot_pure_eb_covariance.py +++ /dev/null @@ -1,88 +0,0 @@ -# %% -"""Visualize Pure E/B covariance matrix structure.""" - -import os -import sys -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns -from mpl_toolkits.axes_grid1 import make_axes_locatable -from plotting_utils import PAPER_MPLSTYLE - -# Unbuffered output for Snakemake log streaming -sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1) -sys.stderr = os.fdopen(sys.stderr.fileno(), "w", buffering=1) - -# Apply paper style -plt.style.use(PAPER_MPLSTYLE) - - -def _load_snakemake(): - if hasattr(sys, "ps1"): - from snakemake_helpers import snakemake_interactive - - return snakemake_interactive( - "results/paper_plots/pure_eb_covariance.png", - str(Path.cwd()), - ) - return snakemake - - -snakemake = _load_snakemake() - -params = snakemake.params - - -def _correlation_from_covariance(cov): - """Convert covariance to correlation matrix.""" - std = np.sqrt(np.diag(cov)) - corr = cov / np.outer(std, std) - return corr - - -def main(): - # Load precomputed data - dataset = np.load(snakemake.input["pure_eb_data"]) - - cov_pure_eb = dataset["cov_pure_eb"] - theta = dataset["theta"] - nbins = len(theta) - - # Create figure - fig, ax = plt.subplots(figsize=(9, 9)) - vlag_cmap = sns.color_palette("vlag", as_cmap=True) - im = ax.matshow(_correlation_from_covariance(cov_pure_eb), cmap=vlag_cmap) - - # Configure tick labels for E/B modes - tick_positions = np.arange(nbins / 2, nbins * 6, nbins) - tick_labels = [ - r"$\xi_+^{\mathrm{E}}$", - r"$\xi_-^{\mathrm{E}}$", - r"$\xi_+^{\mathrm{B}}$", - r"$\xi_-^{\mathrm{B}}$", - r"$\xi_+^{\mathrm{amb}}$", - r"$\xi_-^{\mathrm{amb}}$", - ] - - for ticks in (plt.xticks, plt.yticks): - ticks(tick_positions, tick_labels) - ticks(np.arange(0, nbins * 6 + 1, 20), minor=True) - - ax.tick_params(axis="both", which="major", length=0) - im.set_clim(-1, 1) - divider = make_axes_locatable(ax) - cax = divider.append_axes("right", size="5%", pad=0.1) - plt.colorbar(im, cax=cax) - ax.set_title(f"{params['version']}: semi-analytic correlation matrix") - - # Save - output_path = Path(snakemake.output[0]) - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, dpi=300, bbox_inches="tight") - print(f"Saved plot to {output_path}") - - -if __name__ == "__main__": - main() diff --git a/papers/bmodes/scripts/pure_eb_covariance.py b/papers/bmodes/scripts/pure_eb_covariance.py index ee5ac77c..61b1967a 100644 --- a/papers/bmodes/scripts/pure_eb_covariance.py +++ b/papers/bmodes/scripts/pure_eb_covariance.py @@ -1,4 +1,4 @@ -"""Pure E/B covariance structure claim. +"""Pure E/B covariance structure figure. Validates covariance matrix for B-mode tests by analyzing block structure: - 6 blocks: E+, E-, B+, B-, amb+, amb- @@ -77,7 +77,7 @@ def _cov_to_corr(covariance): return correlation -def main(config, pure_eb_path, out_dir, specs=()): +def main(config, pure_eb_path, out_dir): version = config["fiducial"]["version"] # Load precomputed pure E/B data @@ -176,8 +176,6 @@ def main(config, pure_eb_path, out_dir, specs=()): # Write evidence.json evidence_data = { - "spec_id": "pure_eb_covariance", - **({"spec_path": specs[0]} if specs else {}), "generated": datetime.now().isoformat(), "evidence": { "condition_number": condition_number, @@ -233,16 +231,10 @@ def _from_cli(argv=None): "(provides the 6-block cov_pure_eb)", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument( - "--specs", - nargs="*", - default=[], - help="Optional spec markdown paths recorded in evidence.json for provenance", - ) a = ap.parse_args(argv) with open(a.config) as f: config = yaml.safe_load(f) - main(config, a.pure_eb_data, a.out, specs=a.specs) + main(config, a.pure_eb_data, a.out) if __name__ == "__main__": diff --git a/papers/bmodes/scripts/pure_eb_data_vector.py b/papers/bmodes/scripts/pure_eb_data_vector.py index dbbaeda7..992e1806 100644 --- a/papers/bmodes/scripts/pure_eb_data_vector.py +++ b/papers/bmodes/scripts/pure_eb_data_vector.py @@ -1,4 +1,4 @@ -"""Pure E/B data vector claim (paper Figure 1). +"""Pure E/B data vector figure (paper Figure 1). Fiducial catalog only: pure E/B/ambiguous decomposition of ξ± with B-modes consistent with zero at the fiducial scale cuts. Writes evidence.json with PTE @@ -9,7 +9,7 @@ --config config.yaml \ --pure-eb-data __pure_eb_semianalytic.npz \ --reporting-cov /covariance_processed.txt \ - --out [--specs spec.md ...] + --out """ import argparse @@ -217,7 +217,7 @@ def setup_panel(ax, ylabel_text=None): return fig -def main(config, pure_eb_path, cov_path, out_dir, specs=()): +def main(config, pure_eb_path, cov_path, out_dir): blind = config["fiducial"]["blind"] version = config["fiducial"]["version"] fiducial_xip_scale_cut = tuple(config["fiducial"]["fiducial_xip_scale_cut"]) @@ -297,8 +297,6 @@ def main(config, pure_eb_path, cov_path, out_dir, specs=()): # Write evidence.json (based on leak-corrected fiducial data only) evidence_data = { - "spec_id": "pure_eb_data_vector", - **({"spec_path": specs[0]} if specs else {}), "generated": datetime.now().isoformat(), "evidence": { "fiducial": { @@ -352,16 +350,10 @@ def _from_cli(argv=None): "(covariance_processed.txt) for the total ξ± error bars", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument( - "--specs", - nargs="*", - default=[], - help="Optional spec markdown paths recorded in evidence.json for provenance", - ) a = ap.parse_args(argv) with open(a.config) as f: config = yaml.safe_load(f) - main(config, a.pure_eb_data, a.reporting_cov, a.out, specs=a.specs) + main(config, a.pure_eb_data, a.reporting_cov, a.out) if __name__ == "__main__": diff --git a/papers/bmodes/scripts/pure_eb_version_comparison.py b/papers/bmodes/scripts/pure_eb_version_comparison.py index 8027189f..7b27151b 100644 --- a/papers/bmodes/scripts/pure_eb_version_comparison.py +++ b/papers/bmodes/scripts/pure_eb_version_comparison.py @@ -1,4 +1,4 @@ -"""Pure E/B version comparison claim. +"""Pure E/B version comparison figure. Visualizes total and B-mode correlation functions across catalog versions. Top row: xi_total +/- (same style as data vector plot) @@ -250,13 +250,12 @@ def main( config, results_dir, out_dir, - specs=(), fiducial_version=None, fiducial_pure_eb_data=None, ): plotting_config = config["plotting"] version_labels = plotting_config["version_labels"] - # Leak-corrected, non-ecut versions (matches VERSIONS_LEAK_CORR in claims.smk) + # Leak-corrected versions (matches VERSIONS_LEAK_CORR in figures.smk) versions = [v for v in config["versions"] if "_leak_corr" in v and "_ecut" not in v] blind = config["fiducial"]["blind"] @@ -448,8 +447,6 @@ def main( evidence_versions[f"{v}_{key}"] = int(val) if "dof" in key else float(val) evidence_data = { - "spec_id": "pure_eb_version_comparison", - **({"spec_path": specs[0]} if specs else {}), "generated": datetime.now().isoformat(), "evidence": { "scale_cuts": {k: list(v) for k, v in scale_cuts.items()}, @@ -480,12 +477,6 @@ def _from_cli(argv=None): "__pure_eb_semianalytic.npz files", ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument( - "--specs", - nargs="*", - default=[], - help="Optional spec markdown paths recorded in evidence.json for provenance", - ) ap.add_argument( "--fiducial-version", default=None, @@ -505,7 +496,6 @@ def _from_cli(argv=None): config, a.results_dir, a.out, - specs=a.specs, fiducial_version=a.fiducial_version, fiducial_pure_eb_data=a.fiducial_pure_eb_data, ) diff --git a/papers/bmodes/scripts/run_cl_sweep.py b/papers/bmodes/scripts/run_cl_sweep.py deleted file mode 100644 index d5fd6eb0..00000000 --- a/papers/bmodes/scripts/run_cl_sweep.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Pseudo-Cℓ data-vector + covariance sweep over non-fiducial versions (Design B). - -Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and -runs the same ``generate_pseudo_cl`` / ``generate_pseudo_cl_cov`` compute the -fiducial cl_bb / covariance recipes call, once per version. Each producer writes -its native ``pseudo_cl_{ver}.fits`` / ``pseudo_cl_cov_{ver}.fits`` into ``--out``; -this driver then renames them to the tagged canonical names -``pseudo_cl_{ver}_blind={blind}_{binning}_nbins={nbins}.fits`` and -``pseudo_cl_cov_{ver}_blind={blind}_{binning}_nbins={nbins}.fits`` — the exact -pattern ``cl_version_comparison._pseudo_cl`` / ``._pseudo_cl_cov`` reconstruct. - -Per-version footprint masks resolve internally from cat_config (the version's -``mask:`` entry), so no per-version mask wiring is needed here. Serial over -versions; the estimator uses the recipe's full core allocation per call. - - python run_cl_sweep.py \ - --config .../config.yaml --cat-config .../cat_config.yaml \ - --nside 1024 --npatch 1 --binning powspace --nbins 32 --power 0.5 \ - --blind A --out -""" - -import argparse -import os -import sys - -import yaml - -# generate_pseudo_cl / generate_pseudo_cl_cov live in workflow/scripts; -# sweep_versions is a sibling here. Put workflow/scripts on the path first. -_HERE = os.path.dirname(os.path.abspath(__file__)) -_WSCRIPTS = os.path.abspath( - os.path.join(_HERE, "..", "..", "..", "workflow", "scripts") -) -for _p in (_HERE, _WSCRIPTS): - if _p not in sys.path: - sys.path.insert(0, _p) - -from generate_pseudo_cl import generate_pseudo_cl # noqa: E402 -from generate_pseudo_cl_cov import generate_pseudo_cl_cov # noqa: E402 -from sweep_versions import nonfiducial_versions # noqa: E402 - - -def _canonical(prefix, ver, blind, binning, nbins): - return f"{prefix}_{ver}_blind={blind}_{binning}_nbins={nbins}.fits" - - -def _run_and_tag(producer, prefix, ver, out, blind, binning, nbins, **kw): - """Run one per-version producer, then rename its native FITS to canonical.""" - producer( - version=ver, output_dir=out, blind=blind, binning=binning, nbins=nbins, **kw - ) - native = os.path.join(out, f"{prefix}_{ver}.fits") - canonical = os.path.join(out, _canonical(prefix, ver, blind, binning, nbins)) - os.replace(native, canonical) - print(f"[cl_sweep] {ver}: {os.path.basename(canonical)}") - - -def _from_cli(argv=None): - ap = argparse.ArgumentParser( - description="Pseudo-Cℓ + covariance sweep over the non-fiducial catalog versions." - ) - ap.add_argument( - "--config", required=True, help="Absolute path to bmodes config.yaml" - ) - ap.add_argument( - "--cat-config", required=True, help="Absolute path to cat_config.yaml" - ) - ap.add_argument("--out", required=True, help="Sweep output directory (lc {output})") - ap.add_argument("--versions", nargs="*", default=None, help="Explicit version keys") - ap.add_argument("--nside", type=int, default=1024) - ap.add_argument("--npatch", type=int, default=1) - ap.add_argument( - "--binning", choices=["linear", "logspace", "powspace"], default="powspace" - ) - ap.add_argument("--nbins", type=int, default=32) - ap.add_argument("--power", type=float, default=0.5) - ap.add_argument("--blind", choices=["A", "B", "C"], default="A") - a = ap.parse_args(argv) - with open(a.config) as f: - config = yaml.safe_load(f) - versions = a.versions or nonfiducial_versions(config) - os.makedirs(a.out, exist_ok=True) - shared = dict( - cat_config=a.cat_config, - nside=a.nside, - npatch=a.npatch, - power=a.power, - ) - for ver in versions: - _run_and_tag( - generate_pseudo_cl, - "pseudo_cl", - ver, - a.out, - a.blind, - a.binning, - a.nbins, - **shared, - ) - _run_and_tag( - generate_pseudo_cl_cov, - "pseudo_cl_cov", - ver, - a.out, - a.blind, - a.binning, - a.nbins, - **shared, - ) - - -if __name__ == "__main__": - _from_cli() diff --git a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py deleted file mode 100644 index df3b9b79..00000000 --- a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py +++ /dev/null @@ -1,99 +0,0 @@ -"""COSEBI B-mode PTE-matrix sweep over non-fiducial versions (Design B, Tier-2). - -Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and -runs the same gathered-NPZ COSEBI PTE compute the fiducial cosebis_pte_per_cut -recipe calls (``compute_cosebis_pte_single.main``), once per version. The theta -grid, nmodes and per-pair computation are version-independent (read from -``config["fiducial"]``); only the input xi/cov and the output tag change, so each -sweep call is bit-identical to the fiducial call save for those. Per version the -driver reads that version's 1000-bin integration ξ_± from the xi_sweep output dir -and its Gaussian integration covariance from the cov_sweep output dir (both by -absolute path — lc does not wire cross-output deps, so run xi_sweep + cov_sweep -first), emitting the canonical ``cosebis_ptes_{ver}_{blind}.npz`` — the same -gathered layout the fiducial single-output writes, which -config_space_pte_matrices.py adapts via ``_cosebis_matrix_from_npz`` — into -``--out``. Serial over versions (~25 min/version, 206 pairs). - - python run_cosebis_ptes_sweep.py \ - --config .../config.yaml \ - --xi-sweep-dir \ - --cov-sweep-dir \ - --out [--blind A] [--versions v1 v2 ...] -""" - -import argparse -import os -import sys - -import yaml - -# compute_cosebis_pte_single, plotting_utils, and sweep_versions are siblings -# here; other compute modules live in workflow/scripts. Put both on the path. -_HERE = os.path.dirname(os.path.abspath(__file__)) -_WSCRIPTS = os.path.abspath( - os.path.join(_HERE, "..", "..", "..", "workflow", "scripts") -) -for _p in (_HERE, _WSCRIPTS): - if _p not in sys.path: - sys.path.insert(0, _p) - -from compute_cosebis_pte_single import main as compute_cosebis_pte # noqa: E402 -from sweep_versions import nonfiducial_versions # noqa: E402 - - -def _xi_integration(xi_sweep_dir, ver): - return os.path.join( - xi_sweep_dir, f"{ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" - ) - - -def _cov_integration(cov_sweep_dir, ver, blind): - base = f"covariance_{ver}_{blind}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" - return os.path.join(cov_sweep_dir, base, f"{base}_processed.txt") - - -def _from_cli(argv=None): - ap = argparse.ArgumentParser( - description="Gathered COSEBI B-mode PTE-matrix sweep over the non-fiducial catalog versions." - ) - ap.add_argument( - "--config", required=True, help="Absolute path to bmodes config.yaml" - ) - ap.add_argument( - "--xi-sweep-dir", - required=True, - help="xi_sweep output dir with per-version 1000-bin integration xi_pm .txt", - ) - ap.add_argument( - "--cov-sweep-dir", - required=True, - help="cov_sweep output dir with per-version {base}/{base}_processed.txt cov", - ) - ap.add_argument("--out", required=True, help="Sweep output directory (lc {output})") - ap.add_argument("--blind", default="A", help="Blind tag (paper: A)") - ap.add_argument("--versions", nargs="*", default=None, help="Explicit version keys") - a = ap.parse_args(argv) - - with open(a.config) as f: - config = yaml.safe_load(f) - versions = a.versions or nonfiducial_versions(config) - os.makedirs(a.out, exist_ok=True) - - for ver in versions: - xi_int = _xi_integration(a.xi_sweep_dir, ver) - cov_int = _cov_integration(a.cov_sweep_dir, ver, a.blind) - for f in (xi_int, cov_int): - if not os.path.isfile(f): - raise FileNotFoundError(f"MISSING upstream input for {ver}: {f}") - print(f"[cosebis_ptes_sweep] {ver}", flush=True) - compute_cosebis_pte(config, xi_int, cov_int, a.out, version=ver, blind=a.blind) - print( - f"[cosebis_ptes_sweep] {ver} -> " - f"{os.path.join(a.out, f'cosebis_ptes_{ver}_{a.blind}.npz')}", - flush=True, - ) - print(f"[cosebis_ptes_sweep] done -> {a.out}", flush=True) - - -if __name__ == "__main__": - _from_cli() diff --git a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh deleted file mode 100644 index 0e7ecdf5..00000000 --- a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Pure E/B PTE-matrix sweep over non-fiducial versions (Design B, Tier-2). -# -# Loops the non-fiducial version list (resolved via sweep_versions.py) and runs -# calculate_pure_eb_ptes.py once per version — the same χ² PTE-matrix compute the -# fiducial pure_eb_pte_per_cut recipe calls. Each version's gathered semi-analytic -# NPZ (data vectors + MC covariance) is read from the pure_eb_sweep output dir by -# absolute path — lc does not wire cross-output deps, so the driver reads the -# upstream sweep directly; run pure_eb_sweep before this. Each version emits the -# canonical ``{ver}_{blind}_pure_eb_ptes.npz`` — the exact name -# config_space_pte_matrices.py reconstructs from --pte-intermediate-dir — straight -# into --out. Serial over versions; each version is fast (206-pair grid, ~seconds). -# -# Usage: -# run_pure_eb_ptes_sweep.sh --config --cat-config \ -# --pure-eb-sweep-dir \ -# --out [--blind A] [--versions "v1 v2 ..."] -set -euo pipefail - -. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" - -CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" -while [ $# -gt 0 ]; do - case "$1" in - --config) CONFIG="$2"; shift 2;; - --cat-config) CATCONFIG="$2"; shift 2;; - --pure-eb-sweep-dir) PUREEBSWEEP="$2"; shift 2;; - --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; - --versions) VERSIONS="$2"; shift 2;; - *) echo "unknown arg: $1" >&2; exit 2;; - esac -done - -mkdir -p "$OUT" - -VERSIONS=$(sweep_versions "$CONFIG") - -for ver in $VERSIONS; do - pureeb="$PUREEBSWEEP/${ver}_${BLIND}_pure_eb_semianalytic.npz" - [ -f "$pureeb" ] || { echo "MISSING upstream input for $ver: $pureeb" >&2; exit 1; } - echo "[pure_eb_ptes_sweep] $ver" - spv_python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ - --version "$ver" --blind "$BLIND" \ - --pure-eb-data "$pureeb" --n-samples 2000 --out "$OUT" - echo "[pure_eb_ptes_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_ptes.npz" -done -echo "[pure_eb_ptes_sweep] done -> $OUT" diff --git a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh deleted file mode 100644 index 472cd81d..00000000 --- a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Pure E/B semi-analytic covariance driver (lc-native, container:none recipe). -# -# Runs the 20 independent MC chunks in parallel (each a fresh apptainer-exec -# process, deterministic seed 42+chunk_id) throttled to the node core count, -# then gathers them into the per-(version,blind) semianalytic .npz. Bit-exact -# to the paper's scatter-gather (same per-chunk seeds/order); no nested dask. -# -# Usage: -# run_pure_eb_semianalytic.sh --version SP_v1.4.6.3_leak_corr --blind A \ -# --cat-config \ -# --xi-reporting --xi-integration \ -# --cov-integration \ -# --out [--n-chunks 20] [--n-samples 2000] [--nproc 16] -set -euo pipefail - -. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" - -VERSION=""; BLIND="A"; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" -NCHUNKS=20; NSAMPLES=2000; NPROC="${SLURM_CPUS_PER_TASK:-16}" -MINSEP=1.0; MAXSEP=250.0; NBINS=20 -MINSEPINT=0.5; MAXSEPINT=300.0; NBINSINT=1000; NPATCH=1 - -while [ $# -gt 0 ]; do - case "$1" in - --version) VERSION="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; - --cat-config) CATCONFIG="$2"; shift 2;; - --xi-reporting) XIREP="$2"; shift 2;; - --xi-integration) XIINT="$2"; shift 2;; - --cov-integration) COVINT="$2"; shift 2;; - --out) OUT="$2"; shift 2;; - --n-chunks) NCHUNKS="$2"; shift 2;; - --n-samples) NSAMPLES="$2"; shift 2;; - --nproc) NPROC="$2"; shift 2;; - --min-sep) MINSEP="$2"; shift 2;; - --max-sep) MAXSEP="$2"; shift 2;; - --nbins) NBINS="$2"; shift 2;; - --min-sep-int) MINSEPINT="$2"; shift 2;; - --max-sep-int) MAXSEPINT="$2"; shift 2;; - --nbins-int) NBINSINT="$2"; shift 2;; - --npatch) NPATCH="$2"; shift 2;; - *) echo "unknown arg: $1" >&2; exit 2;; - esac -done - -mkdir -p "$OUT/chunks" - -echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION blind=$BLIND" -for i in $(seq 0 $((NCHUNKS-1))); do - ( - SPV_EXEC_EXTRA=$SINGLE_THREAD_ENV - spv_python "$PSCRIPTS/precompute_pure_eb_chunk.py" \ - --chunk-id "$i" --n-chunks "$NCHUNKS" --n-samples "$NSAMPLES" \ - --version "$VERSION" --blind "$BLIND" --cat-config "$CATCONFIG" \ - --xi-reporting "$XIREP" --xi-integration "$XIINT" --cov-integration "$COVINT" \ - --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" \ - --min-sep-int "$MINSEPINT" --max-sep-int "$MAXSEPINT" --nbins-int "$NBINSINT" \ - --npatch "$NPATCH" --out "$OUT/chunks" - ) > "$OUT/chunks/chunk_${i}.log" 2>&1 & - while [ "$(jobs -rp | wc -l)" -ge "$NPROC" ]; do sleep 1; done -done -wait - -# Verify all chunks landed -missing=0 -for i in $(seq 0 $((NCHUNKS-1))); do - [ -f "$OUT/chunks/pure_eb_chunk_${i}.npz" ] || { echo "MISSING chunk $i (see $OUT/chunks/chunk_${i}.log)" >&2; missing=1; } -done -[ "$missing" -eq 0 ] || { echo "[pure_eb] chunk failures — aborting gather" >&2; exit 1; } -echo "[pure_eb] all $NCHUNKS chunks done; gathering" - -spv_python "$PSCRIPTS/gather_pure_eb_chunks.py" \ - --version "$VERSION" --blind "$BLIND" \ - --xi-reporting "$XIREP" --xi-integration "$XIINT" \ - --chunks-dir "$OUT/chunks" \ - --min-sep "$MINSEP" --max-sep "$MAXSEP" --nbins "$NBINS" \ - --min-sep-int "$MINSEPINT" --max-sep-int "$MAXSEPINT" --nbins-int "$NBINSINT" \ - --npatch "$NPATCH" --out "$OUT" -echo "[pure_eb] done -> $OUT" diff --git a/papers/bmodes/scripts/run_pure_eb_sweep.sh b/papers/bmodes/scripts/run_pure_eb_sweep.sh deleted file mode 100755 index 8195f150..00000000 --- a/papers/bmodes/scripts/run_pure_eb_sweep.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# Pure E/B semi-analytic covariance sweep over non-fiducial versions (Design B). -# -# Loops the non-fiducial version list (resolved via sweep_versions.py) and runs -# the same run_pure_eb_semianalytic.sh scatter-gather the fiducial -# pure_eb_semianalytic_data recipe calls, once per version. Each version's -# reporting+integration ξ± is read from the xi_sweep output dir and its Gaussian -# integration covariance from the cov_sweep output dir (both by absolute path — -# lc does not wire cross-output deps, so the driver reads upstream sweeps -# directly; run xi_sweep + cov_sweep before this). The gathered -# ``{ver}_{blind}_pure_eb_semianalytic.npz`` — already the canonical name -# pure_eb_version_comparison (--results-dir) reads — is moved into --out; the -# per-version MC chunks stage in a scratch subdir that is removed after gather. -# -# Usage: -# run_pure_eb_sweep.sh --config --cat-config \ -# --xi-sweep-dir \ -# --cov-sweep-dir \ -# --out [--blind A] [--versions "v1 v2 ..."] -set -euo pipefail - -. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" - -CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" -while [ $# -gt 0 ]; do - case "$1" in - --config) CONFIG="$2"; shift 2;; - --cat-config) CATCONFIG="$2"; shift 2;; - --xi-sweep-dir) XISWEEP="$2"; shift 2;; - --cov-sweep-dir) COVSWEEP="$2"; shift 2;; - --out) OUT="$2"; shift 2;; - --blind) BLIND="$2"; shift 2;; - --versions) VERSIONS="$2"; shift 2;; - *) echo "unknown arg: $1" >&2; exit 2;; - esac -done - -mkdir -p "$OUT" - -VERSIONS=$(sweep_versions "$CONFIG") - -for ver in $VERSIONS; do - xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt" - xiint="$XISWEEP/${ver}_xi_minsep=0.5_maxsep=300.0_nbins=1000_npatch=1.txt" - covbase="covariance_${ver}_${BLIND}_g_minsep=0.5_maxsep=300.0_nbins=1000_masked" - covint="$COVSWEEP/$covbase/${covbase}_processed.txt" - for f in "$xirep" "$xiint" "$covint"; do - [ -f "$f" ] || { echo "MISSING upstream input for $ver: $f" >&2; exit 1; } - done - echo "[pure_eb_sweep] $ver" - stage="$OUT/_stage_${ver}" - bash "$PSCRIPTS/run_pure_eb_semianalytic.sh" \ - --version "$ver" --blind "$BLIND" --cat-config "$CATCONFIG" \ - --xi-reporting "$xirep" --xi-integration "$xiint" --cov-integration "$covint" \ - --out "$stage" - mv "$stage/${ver}_${BLIND}_pure_eb_semianalytic.npz" \ - "$OUT/${ver}_${BLIND}_pure_eb_semianalytic.npz" - rm -rf "$stage" - echo "[pure_eb_sweep] $ver -> $OUT/${ver}_${BLIND}_pure_eb_semianalytic.npz" -done -echo "[pure_eb_sweep] done -> $OUT" diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py deleted file mode 100644 index 211214e3..00000000 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ /dev/null @@ -1,85 +0,0 @@ -"""TreeCorr ξ±(θ) sweep over the non-fiducial catalog versions (Design B). - -Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and -runs the same ``run_2pcf.run_2pcf`` compute the fiducial two_point recipes call, -once per version, writing every version's ξ± text dump into one lc ``{output}`` -dir under run_2pcf's native, already-canonical name -``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.txt`` — the -exact pattern ``cosebis_version_comparison._xi_integration`` reconstructs. - -Both grids are produced per version: the 1000-bin integration grid feeds -``cosebis_version_comparison`` (``--results-dir``) and the pure E/B sweep's -integral kernels; the 20-bin reporting grid feeds the pure E/B sweep's output -binning. Serial over versions — lc's dask handles cross-output concurrency, and -TreeCorr already uses the recipe's full OpenMP allocation per call. - - python run_xi_sweep.py \ - --config .../config.yaml --cat-config .../cat_config.yaml --out -""" - -import argparse -import os -import sys - -import yaml - -# run_2pcf lives in workflow/scripts; sweep_versions is a sibling here. -# Put workflow/scripts on the path first. -_HERE = os.path.dirname(os.path.abspath(__file__)) -_WSCRIPTS = os.path.abspath( - os.path.join(_HERE, "..", "..", "..", "workflow", "scripts") -) -for _p in (_HERE, _WSCRIPTS): - if _p not in sys.path: - sys.path.insert(0, _p) - -from run_2pcf import run_2pcf # noqa: E402 -from sweep_versions import nonfiducial_versions # noqa: E402 - -GRIDS = { - "reporting": dict(min_sep=1.0, max_sep=250.0, nbins=20, npatch=1), - "integration": dict(min_sep=0.5, max_sep=300.0, nbins=1000, npatch=1), -} - - -def _from_cli(argv=None): - ap = argparse.ArgumentParser( - description="TreeCorr ξ± sweep over the non-fiducial catalog versions." - ) - ap.add_argument( - "--config", required=True, help="Absolute path to bmodes config.yaml" - ) - ap.add_argument( - "--cat-config", required=True, help="Absolute path to cat_config.yaml" - ) - ap.add_argument("--out", required=True, help="Sweep output directory (lc {output})") - ap.add_argument( - "--versions", - nargs="*", - default=None, - help="Explicit version keys (default: non-fiducial non-ecut from config)", - ) - ap.add_argument( - "--grids", - nargs="*", - choices=list(GRIDS), - default=list(GRIDS), - help="Which angular grids to measure per version (default: both)", - ) - a = ap.parse_args(argv) - with open(a.config) as f: - config = yaml.safe_load(f) - versions = a.versions or nonfiducial_versions(config) - for ver in versions: - for grid in a.grids: - run_2pcf( - ver=ver, - cat_config=a.cat_config, - output_dir=a.out, - grid=grid, - **GRIDS[grid], - ) - - -if __name__ == "__main__": - _from_cli() diff --git a/papers/bmodes/scripts/sweep_versions.py b/papers/bmodes/scripts/sweep_versions.py deleted file mode 100644 index 2cb10f2d..00000000 --- a/papers/bmodes/scripts/sweep_versions.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Resolve the non-fiducial catalog-version list for the Paper II version sweep. - -The Design-B version sweep recomputes every non-fiducial catalog variant through -lc so the version-comparison figures (paper Figure 5) are reproduced end-to-end -rather than read from the canonical CosmoStat tree. The variant set is the config -``versions:`` list minus the |e|<0.7 ``_ecut`` cross-checks (out of scope for -every comparison filter) and minus the fiducial version (``config['fiducial'] -['version']``), which is produced by the single-output recipes and handed to the -comparison plotters through their ``--fiducial-*`` overrides. - -The remaining 7 are the union per-version product set: the 3 non-fiducial -leak-corrected catalogs feeding the Tier-1 overlay figures, plus the 4 -uncorrected catalogs feeding the Tier-2 appendix PTE matrices. Both the sweep -drivers (as an import) and the bash drivers (as a ``--config`` CLI printing one -version per line) resolve the list here, so there is a single source of truth. -""" - -import argparse - -import yaml - - -def nonfiducial_versions(config): - """Non-fiducial, non-ecut catalog versions from a loaded bmodes config.""" - fiducial = config["fiducial"]["version"] - return [v for v in config["versions"] if "_ecut" not in v and v != fiducial] - - -def _from_cli(argv=None): - ap = argparse.ArgumentParser( - description="Print the non-fiducial sweep version list, one per line." - ) - ap.add_argument( - "--config", required=True, help="Absolute path to bmodes config.yaml" - ) - a = ap.parse_args(argv) - with open(a.config) as f: - config = yaml.safe_load(f) - for v in nonfiducial_versions(config): - print(v) - - -if __name__ == "__main__": - _from_cli() diff --git a/papers/bmodes/scripts/test_bandpower_sampling.py b/papers/bmodes/scripts/test_bandpower_sampling.py deleted file mode 100644 index dec8633a..00000000 --- a/papers/bmodes/scripts/test_bandpower_sampling.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Test whether 32-bin bandpower sampling causes harmonic/config COSEBIS disagreement. - -Controlled experiment: compute COSEBIS from the SAME smooth theory C(ℓ) -via three paths: - -1. Dense ℓ: W_n(ℓ) × C(ℓ) integrated over 10,000 ℓ points (ground truth) -2. 32-bin: W_n(ℓ) × C(ℓ) integrated over the exact 32 powspace bins -3. Config-space: theory ξ±(θ) → cosebis_from_xipm() (independent check) - -If (1) ≈ (3) but (2) ≠ (1), bandpower sampling is the cause. -If (1) ≈ (2), something else is going on. - -Uses a simple power-law C(ℓ) = A × ℓ^α to avoid external dependencies. -Also tests with a more realistic shape: C(ℓ) = A × ℓ^α × exp(-ℓ/ℓ_0). - -Author: Claude Code -""" - -import numpy as np -from cosmo_numba.B_modes.cosebis import COSEBIS -from scipy import special - -# The exact 32 powspace ℓ values from the pipeline -ELL_32 = np.array( - [ - 12.5, - 24.0, - 38.5, - 56.5, - 78.0, - 103.0, - 131.5, - 163.5, - 199.0, - 238.0, - 281.0, - 327.5, - 377.0, - 430.0, - 487.0, - 547.5, - 611.0, - 678.0, - 749.0, - 823.5, - 901.0, - 982.0, - 1067.0, - 1155.5, - 1247.5, - 1343.0, - 1441.5, - 1544.0, - 1650.0, - 1759.5, - 1872.5, - 1988.5, - ] -) - - -def theory_cl(ell, model="power_law"): - """Smooth theory C(ℓ) for testing.""" - if model == "power_law": - # Simple power law, roughly cosmic-shear-like - return 1e-7 * (ell / 100.0) ** (-1.5) - elif model == "realistic": - # Power law with exponential cutoff (more structure) - return 1e-7 * (ell / 100.0) ** (-1.2) * np.exp(-ell / 1500.0) - else: - raise ValueError(f"Unknown model: {model}") - - -def config_space_cosebis(theta_min, theta_max, nmodes, cl_model, n_theta=5_000): - """Compute COSEBIS via config-space path: C(ℓ) → ξ±(θ) → cosebis_from_xipm. - - ξ+(θ) = (1/2π) ∫ dℓ ℓ C(ℓ) J_0(ℓθ) - ξ-(θ) = (1/2π) ∫ dℓ ℓ C(ℓ) J_4(ℓθ) - """ - # Dense ℓ grid for Hankel transform - n_ell = 20_000 - ell_dense = np.logspace(np.log10(1), np.log10(1e5), n_ell) - cl = theory_cl(ell_dense, model=cl_model) - - # θ grid (log-spaced in arcmin, clipped to avoid float precision issues) - theta_arcmin = np.logspace(np.log10(theta_min), np.log10(theta_max), n_theta) - theta_arcmin = np.clip(theta_arcmin, theta_min, theta_max) - theta_rad = np.deg2rad(theta_arcmin / 60.0) - - # Vectorized Hankel transform: (n_theta, n_ell) outer product - # Process in chunks to avoid memory blow-up - chunk = 500 - xip = np.zeros(n_theta) - xim = np.zeros(n_theta) - weight = ell_dense * cl / (2 * np.pi) # (n_ell,) - - for start in range(0, n_theta, chunk): - end = min(start + chunk, n_theta) - t_chunk = theta_rad[start:end, None] # (chunk, 1) - x = ell_dense[None, :] * t_chunk # (chunk, n_ell) - xip[start:end] = np.trapezoid(weight * special.j0(x), ell_dense, axis=1) - xim[start:end] = np.trapezoid(weight * special.jv(4, x), ell_dense, axis=1) - - # COSEBIS from ξ± - cosebis = COSEBIS(theta_min, theta_max, nmodes) - ce, cb = cosebis.cosebis_from_xipm(theta_arcmin, xip, xim) - return ce, cb - - -def harmonic_cosebis(ell, cl_E, theta_min, theta_max, nmodes): - """Compute COSEBIS via harmonic path: C(ℓ) → cosebis_from_Cell.""" - cosebis = COSEBIS(theta_min, theta_max, nmodes) - cl_B = np.zeros_like(cl_E) - ce, cb = cosebis.cosebis_from_Cell(ell, cl_E, cl_B) - return ce, cb - - -def make_powspace_bins(ell_min, ell_max, nbins, power=0.5): - """Generate powspace bin centres matching NaMaster convention.""" - edges = np.linspace(ell_min**power, ell_max**power, nbins + 1) ** (1.0 / power) - return 0.5 * (edges[:-1] + edges[1:]) - - -def main(): - nmodes = 20 - bin_counts = [32, 64, 100, 200, 500] - - for cl_model in ["power_law", "realistic"]: - for theta_min, theta_max, label in [ - (1.0, 250.0, "full [1,250]'"), - (12.0, 83.0, "fiducial [12,83]'"), - ]: - print(f"\n{'=' * 75}") - print(f"C(ℓ) model: {cl_model}, scale cut: {label}") - print(f"{'=' * 75}") - - # --- Ground truth: dense ℓ --- - ell_dense = np.logspace(np.log10(2), np.log10(5000), 10_000) - cl_dense = theory_cl(ell_dense, model=cl_model) - ce_dense, _ = harmonic_cosebis( - ell_dense, - cl_dense, - theta_min, - theta_max, - nmodes, - ) - - # --- Config-space reference --- - print("Computing config-space reference (Hankel + xipm)...") - ce_config, _ = config_space_cosebis( - theta_min, - theta_max, - nmodes, - cl_model, - ) - - # --- Binned paths --- - ce_binned = {} - for nb in bin_counts: - ell_b = make_powspace_bins(8, 2100, nb) - cl_b = theory_cl(ell_b, model=cl_model) - ce_b, _ = harmonic_cosebis(ell_b, cl_b, theta_min, theta_max, nmodes) - ce_binned[nb] = ce_b - - # --- Table: ratio to dense for each bin count --- - hdr = f"{'mode':>5}" - for nb in bin_counts: - hdr += f" {nb:>8d}-bin" - hdr += " dense/cfg" - print(f"\n{hdr}") - print("-" * (14 + 13 * len(bin_counts))) - - for n in range(nmodes): - line = f" n={n + 1:>2d}" - for nb in bin_counts: - r = ( - ce_binned[nb][n] / ce_dense[n] - if abs(ce_dense[n]) > 1e-30 - else np.nan - ) - line += f" {r:>12.4f}" - r_cfg = ( - ce_dense[n] / ce_config[n] if abs(ce_config[n]) > 1e-30 else np.nan - ) - line += f" {r_cfg:>12.4f}" - print(line) - - # Summary: max error over modes 1-6 (where signal is meaningful) - n_summary = min(6, nmodes) - print(f"\n Max |1 - ratio| for modes 1-{n_summary} (ratio to dense):") - for nb in bin_counts: - ratios = ce_binned[nb][:n_summary] / ce_dense[:n_summary] - valid = np.isfinite(ratios) - if valid.any(): - err = np.max(np.abs(1 - ratios[valid])) - print(f" {nb:>4d} bins: {err:.4f} ({err * 100:.1f}%)") - ratios_cfg = ce_dense[:n_summary] / ce_config[:n_summary] - valid = np.isfinite(ratios_cfg) - if valid.any(): - err = np.max(np.abs(1 - ratios_cfg[valid])) - print(f" dense/config: {err:.4f} ({err * 100:.1f}%)") - - -if __name__ == "__main__": - main() diff --git a/papers/bmodes/scripts/update_survey_stats.py b/papers/bmodes/scripts/update_survey_stats.py deleted file mode 100644 index a2dc386c..00000000 --- a/papers/bmodes/scripts/update_survey_stats.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Recompute survey statistics (A, n_e, sigma_e) for all active versions. - -Uses the new footprint masks to define the survey area, then computes -n_eff and sigma_e from each version's shear catalog in chunks to limit -memory usage. - -Usage: - python papers/bmodes/scripts/update_survey_stats.py \ - --mask-standard --mask-starhalo -""" - -import argparse -import gc -import os -from pathlib import Path - -import healpy as hp -import numpy as np -import yaml -from astropy.io import fits - -os.chdir("/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val") - - -# Versions that use the standard footprint mask -STANDARD_VERSIONS = [ - "SP_v1.4.5", - "SP_v1.4.6", - "SP_v1.4.6_ecut07", - "SP_v1.4.11.3", - "SP_v1.4.11.3_ecut07", -] - -# Versions that use the star-halo footprint mask -STARHALO_VERSIONS = [ - "SP_v1.4.8", -] - - -def area_from_mask(mask_path): - """Compute survey area from a binary HEALPix mask.""" - mask = hp.read_map(mask_path, dtype=np.float64) - return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) - - -def compute_stats_chunked(catalog_path, w_col, e1_col, e2_col, chunk_size=1_000_000): - """Compute sum_w, sum_w2, sum_w2_e2 in chunks to limit memory.""" - with fits.open(catalog_path, memmap=True) as hdul: - data = hdul[1].data - nrows = len(data) - - sum_w = 0.0 - sum_w2 = 0.0 - sum_w2_e2 = 0.0 - - for start in range(0, nrows, chunk_size): - stop = min(start + chunk_size, nrows) - w = np.asarray(data[w_col][start:stop], dtype=np.float64) - e1 = np.asarray(data[e1_col][start:stop], dtype=np.float64) - e2 = np.asarray(data[e2_col][start:stop], dtype=np.float64) - - sum_w += np.sum(w) - w2 = w**2 - sum_w2 += np.sum(w2) - sum_w2_e2 += np.sum(w2 * (e1**2 + e2**2)) - - del w, e1, e2, w2 - - return float(sum_w), float(sum_w2), float(sum_w2_e2), nrows - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--mask-standard", required=True) - parser.add_argument("--mask-starhalo", required=True) - parser.add_argument("--dry-run", action="store_true") - args = parser.parse_args() - - # Load cat_config - config_path = Path("cat_config.yaml") - with open(config_path) as f: - cc = yaml.safe_load(f) - - # Compute areas from masks (only two masks to load) - area_standard = area_from_mask(args.mask_standard) - area_starhalo = area_from_mask(args.mask_starhalo) - print(f"Standard footprint area: {area_standard:.2f} deg²") - print(f"Star-halo footprint area: {area_starhalo:.2f} deg²\n") - - all_versions = [(v, area_standard) for v in STANDARD_VERSIONS] + [ - (v, area_starhalo) for v in STARHALO_VERSIONS - ] - - print( - f"{'Version':<30} {'Area (deg²)':>12} {'n_eff':>10} {'sigma_e':>10} {'Old A':>10} {'Old n_e':>10} {'Old σ_e':>10}" - ) - print("-" * 105) - - for ver, area_deg2 in all_versions: - if ver not in cc: - print(f"{ver:<30} SKIPPED (not in config)") - continue - - shear_cfg = cc[ver].get("shear", {}) - if "path" not in shear_cfg: - print(f"{ver:<30} SKIPPED (no shear path)") - continue - - subdir = cc[ver].get("subdir", "") - catalog_path = shear_cfg["path"] - if not os.path.isabs(catalog_path): - catalog_path = os.path.join(subdir, catalog_path) - - w_col = shear_cfg.get("w_col", "w") - e1_col = shear_cfg.get("e1_col", "e1") - e2_col = shear_cfg.get("e2_col", "e2") - - # Get old values for comparison - old_cov = cc[ver].get("cov_th", {}) - old_A = old_cov.get("A", 0) - old_ne = old_cov.get("n_e", 0) - old_se = old_cov.get("sigma_e", 0) - - sum_w, sum_w2, sum_w2_e2, nrows = compute_stats_chunked( - catalog_path, w_col, e1_col, e2_col - ) - - area_arcmin2 = area_deg2 * 3600.0 - n_eff = (sum_w**2) / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 - sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 - - print( - f"{ver:<30} {area_deg2:>12.2f} {n_eff:>10.6f} {sigma_e:>10.6f} {old_A:>10.2f} {old_ne:>10.6f} {old_se:>10.6f}" - ) - - if not args.dry_run: - if "cov_th" not in cc[ver]: - cc[ver]["cov_th"] = {} - cc[ver]["cov_th"]["A"] = float(area_deg2) - cc[ver]["cov_th"]["n_e"] = float(n_eff) - cc[ver]["cov_th"]["sigma_e"] = float(sigma_e) - - gc.collect() - - if not args.dry_run: - with open(config_path, "w") as f: - yaml.dump(cc, f, sort_keys=False) - print("\ncat_config.yaml updated.") - else: - print("\nDry run — no changes written.") - - -if __name__ == "__main__": - main() diff --git a/papers/bmodes/scripts/validate_cosebis_filters.py b/papers/bmodes/scripts/validate_cosebis_filters.py deleted file mode 100644 index cddbf421..00000000 --- a/papers/bmodes/scripts/validate_cosebis_filters.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Validate COSEBIS filter functions W_n(ℓ) against direct integration. - -The cosmo_numba code computes W_n(ℓ) via FFT-log (Hankel transform of T_+^log). -This script validates that against direct numerical integration: - - W_n(ℓ) = ∫ dθ θ T_+^log(θ) J_0(ℓθ) - -where θ is in radians and T_+^log is the log-basis filter function. - -Tests both full [1,250]' and fiducial [12,83]' scale cuts, -for modes n = 1 through 20, across a dense ℓ grid. - -Author: Claude Code -""" - -import matplotlib.pyplot as plt -import numpy as np -from cosmo_numba.B_modes.cosebis import COSEBIS -from scipy import integrate, special - - -def direct_Wn_integration(ell_val, theta_arcmin, Tp): - """Compute W_n(ℓ) via direct numerical integration. - - W_n(ℓ) = ∫ dθ θ T_+^log(θ) J_0(ℓθ) - - Uses trapezoidal rule on a fine theta grid. - """ - theta_rad = np.deg2rad(theta_arcmin / 60) - j0 = special.j0(ell_val * theta_rad) - integrand = theta_rad * Tp * j0 - return np.trapezoid(integrand, theta_rad) - - -def direct_Wn_quad(ell_val, theta_arcmin, Tp): - """Compute W_n(ℓ) via adaptive quadrature (gold standard). - - Interpolates T_+^log and uses scipy.integrate.quad. - """ - from scipy.interpolate import CubicSpline - - theta_rad = np.deg2rad(theta_arcmin / 60) - cs = CubicSpline(theta_rad, Tp) - - def integrand(t): - return t * cs(t) * special.j0(ell_val * t) - - result, _ = integrate.quad( - integrand, - theta_rad[0], - theta_rad[-1], - limit=500, - epsrel=1e-10, - epsabs=0, - ) - return result - - -def validate_scale_cut(theta_min, theta_max, nmodes, ell_test, n_theta=200_000): - """Run validation for one scale cut. - - Returns dict with keys 'fftlog', 'direct', 'ratio' — each (nmodes, n_ell). - """ - print(f"\n{'=' * 70}") - print(f"Scale cut: [{theta_min}, {theta_max}] arcmin, modes 1-{nmodes}") - print(f"{'=' * 70}") - - cosebis = COSEBIS(theta_min, theta_max, nmodes) - - # Fine log-spaced θ grid for direct integration - theta = np.logspace(np.log10(theta_min), np.log10(theta_max), n_theta) - - print("Computing T_+^log filters...") - Tp = cosebis.get_Tp_log(theta) - - # FFT-log W_n(ℓ) — pass ell directly, let it use 100k internal grid - print("Computing W_n(ℓ) via FFT-log (100k grid + padding)...") - Wn_fftlog = cosebis.get_Wn_log(ell_test) - - # Direct integration at each ℓ - print(f"Computing W_n(ℓ) via direct integration ({n_theta} θ points)...") - n_ell = len(ell_test) - Wn_direct = np.zeros((nmodes, n_ell)) - for n in range(nmodes): - for i, ell in enumerate(ell_test): - Wn_direct[n, i] = direct_Wn_integration(ell, theta, Tp[n]) - - # Ratio - with np.errstate(divide="ignore", invalid="ignore"): - ratio = np.where( - np.abs(Wn_direct) > 1e-30, - Wn_fftlog / Wn_direct, - np.nan, - ) - - return { - "fftlog": Wn_fftlog, - "direct": Wn_direct, - "ratio": ratio, - "Tp": Tp, - "theta": theta, - } - - -def print_table(ell_test, result, nmodes, indices=None, spot_modes=None): - """Print comparison table for selected modes at selected ℓ indices.""" - if spot_modes is None: - spot_modes = list(range(min(nmodes, 6))) - if indices is None: - indices = list(range(len(ell_test))) - - for n in spot_modes: - print(f"\n--- Mode n={n + 1} ---") - print(f"{'ell':>8} {'FFT-log':>14} {'Direct':>14} {'Ratio':>10}") - print("-" * 50) - - for i in indices: - r = result["ratio"][n, i] - print( - f"{ell_test[i]:>8.0f} " - f"{result['fftlog'][n, i]:>14.6e} " - f"{result['direct'][n, i]:>14.6e} " - f"{r:>10.4f}" - ) - - -def quad_spot_check(theta_min, theta_max, nmodes, ell_spot): - """Spot-check a few (mode, ℓ) values with adaptive quadrature.""" - print(f"\n{'=' * 70}") - print(f"Adaptive quadrature spot check: [{theta_min}, {theta_max}]'") - print(f"{'=' * 70}") - - cosebis = COSEBIS(theta_min, theta_max, nmodes) - theta_fine = np.logspace(np.log10(theta_min), np.log10(theta_max), 500_000) - Tp = cosebis.get_Tp_log(theta_fine) - - Wn_fftlog = cosebis.get_Wn_log(np.array(ell_spot, dtype=float)) - - print( - f"{'mode':>6} {'ell':>8} {'FFT-log':>14} {'quad':>14} {'trapz(500k)':>14} {'FFT/quad':>10} {'trapz/quad':>10}" - ) - print("-" * 80) - - for n in range(min(nmodes, 5)): - for j, ell in enumerate(ell_spot): - wn_fft = Wn_fftlog[n, j] - wn_quad = direct_Wn_quad(ell, theta_fine, Tp[n]) - wn_trapz = direct_Wn_integration(ell, theta_fine, Tp[n]) - r_fft = wn_fft / wn_quad if abs(wn_quad) > 1e-30 else np.nan - r_trapz = wn_trapz / wn_quad if abs(wn_quad) > 1e-30 else np.nan - print( - f"{n + 1:>6d} {ell:>8.0f} {wn_fft:>14.6e} {wn_quad:>14.6e} " - f"{wn_trapz:>14.6e} {r_fft:>10.6f} {r_trapz:>10.6f}" - ) - - -def plot_results(ell_test, results, output_path): - r"""Plot W_n ratio (FFT-log / direct) for both scale cuts.""" - fig, axes = plt.subplots(2, 1, figsize=(12, 10), sharex=True) - - for ax, (label, result) in zip(axes, results.items()): - nmodes = result["ratio"].shape[0] - cmap = plt.get_cmap("viridis", nmodes) - - for n in range(nmodes): - valid = np.isfinite(result["ratio"][n]) - ax.plot( - ell_test[valid], - result["ratio"][n, valid], - color=cmap(n), - alpha=0.7, - lw=1.2, - label=f"n={n + 1}" if n < 10 else None, - ) - - ax.axhline(1.0, color="k", ls="--", lw=0.8) - ax.axhspan(0.99, 1.01, color="green", alpha=0.1) - ax.set_ylabel("FFT-log / Direct") - ax.set_title(rf"$W_n(\ell)$ accuracy: {label}") - ax.set_ylim(0.9, 1.1) - ax.legend(ncol=5, fontsize=7, loc="lower left") - - axes[1].set_xlabel(r"$\ell$") - axes[1].set_xscale("log") - fig.tight_layout() - fig.savefig(output_path, dpi=150, bbox_inches="tight") - print(f"\nFigure saved: {output_path}") - plt.close(fig) - - -def main(): - print("COSEBIS W_n(ℓ) Filter Validation") - print("FFT-log (cosmo_numba, post-Jan 23 rewrite) vs direct integration") - - nmodes = 20 - - # Dense ℓ grid covering the pipeline range - ell_test = np.logspace(np.log10(12), np.log10(3000), 200) - - spot_idx = [0, 25, 50, 75, 100, 125, 150, 175, 199] - - # --- Full scale cut [1, 250]' --- - result_full = validate_scale_cut(1.0, 250.0, nmodes, ell_test) - print_table(ell_test, result_full, nmodes, indices=spot_idx) - - # --- Fiducial scale cut [12, 83]' --- - result_fid = validate_scale_cut(12.0, 83.0, nmodes, ell_test) - print_table(ell_test, result_fid, nmodes, indices=spot_idx) - - # --- Summary statistics --- - for label, result in [ - ("Full [1,250]'", result_full), - ("Fiducial [12,83]'", result_fid), - ]: - print(f"\n{'=' * 70}") - print(f"Summary: {label}") - print(f"{'=' * 70}") - for n in range(nmodes): - valid = np.isfinite(result["ratio"][n]) - if valid.any(): - r = result["ratio"][n, valid] - print( - f" n={n + 1:>2d}: ratio range [{r.min():.4f}, {r.max():.4f}], " - f"median={np.median(r):.4f}, |1-ratio| max={np.max(np.abs(1 - r)):.4f}" - ) - - # --- Adaptive quadrature spot checks --- - ell_spot = [50, 200, 500, 1000, 2000] - quad_spot_check(1.0, 250.0, 5, ell_spot) - quad_spot_check(12.0, 83.0, 5, ell_spot) - - # --- Plot --- - results = { - "[1, 250]' (full)": result_full, - "[12, 83]' (fiducial)": result_fid, - } - output_path = "workflow/scripts/validate_cosebis_filters_results.png" - plot_results(ell_test, results, output_path) - - # --- Verdict --- - all_ratios = np.concatenate( - [ - result_full["ratio"][np.isfinite(result_full["ratio"])], - result_fid["ratio"][np.isfinite(result_fid["ratio"])], - ] - ) - max_err = np.max(np.abs(1 - all_ratios)) - print(f"\n{'=' * 70}") - print("VERDICT") - print(f"{'=' * 70}") - print(f"Maximum |1 - ratio| across all modes, scale cuts, ℓ: {max_err:.6f}") - if max_err < 0.01: - print("PASS: FFT-log W_n(ℓ) accurate to <1%") - print(" → Harmonic/config disagreement must come from elsewhere") - print(" (bandpower sampling, mask, noise bias)") - elif max_err < 0.05: - print("MARGINAL: FFT-log W_n(ℓ) accurate to <5% but >1%") - print(" → May contribute to harmonic/config disagreement") - else: - print("FAIL: FFT-log W_n(ℓ) inaccurate (>5% error)") - print(" → File upstream issue on cosmo_numba") - - -if __name__ == "__main__": - main() diff --git a/papers/bmodes/scripts/validate_cosebis_theory.py b/papers/bmodes/scripts/validate_cosebis_theory.py deleted file mode 100644 index dda26f34..00000000 --- a/papers/bmodes/scripts/validate_cosebis_theory.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Validate COSEBIS integration: harmonic vs config-space using theory. - -Uses CCL theory C_ℓ and ξ±(θ) which are mathematically related by exact Hankel -transforms. If the COSEBIS code is correct, E_n computed from either method -should agree to numerical precision. - -This is a clean validation that doesn't depend on mock data consistency. - -Author: Claude Code -""" - -import numpy as np -import pyccl as ccl -from cosmo_numba.B_modes.cosebis import COSEBIS - - -def setup_cosmology_and_tracer(): - """Setup CCL cosmology and weak lensing tracer.""" - cosmo = ccl.Cosmology( - Omega_c=0.27, - Omega_b=0.045, - h=0.67, - sigma8=0.8, - n_s=0.96, - transfer_function="boltzmann_camb", - matter_power_spectrum="halofit", - ) - - # Simple Gaussian n(z) centered at z=1.0 - z = np.linspace(0.01, 3.0, 300) - nz = np.exp(-0.5 * ((z - 1.0) / 0.2) ** 2) - nz /= np.trapezoid(nz, z) - - tracer = ccl.WeakLensingTracer(cosmo, dndz=(z, nz)) - - return cosmo, tracer - - -def get_theory_cls(ell, cosmo, tracer): - """Compute theory E-mode C_ℓ.""" - cl_ee = ccl.angular_cl(cosmo, tracer, tracer, ell) - cl_bb = np.zeros_like(cl_ee) - return cl_ee, cl_bb - - -def get_theory_xipm(cosmo, ell, cl_ee, theta_arcmin): - """Compute ξ±(θ) from C_ℓ using CCL's built-in correlation function.""" - theta_rad = np.deg2rad(theta_arcmin / 60) - - xip = ccl.correlation(cosmo, ell=ell, C_ell=cl_ee, theta=theta_rad, type="GG+") - xim = ccl.correlation(cosmo, ell=ell, C_ell=cl_ee, theta=theta_rad, type="GG-") - - return xip, xim - - -def main(): - print("=" * 60) - print("COSEBIS Theory Validation: Harmonic vs Config-space") - print("=" * 60) - - # Parameters matching real data analysis - theta_min = 1.0 # arcmin - theta_max = 250.0 # arcmin - nmodes = 6 - - # Fine grids for integration - # Use integer ell for CCL compatibility, but float64 for numba - ell = np.unique(np.geomspace(2, 30000, 2000).astype(int)).astype(np.float64) - n_theta = 1000 - theta_integration = np.logspace(np.log10(theta_min), np.log10(theta_max), n_theta) - - print("\nParameters:") - print(f" theta range: [{theta_min}, {theta_max}] arcmin") - print(f" COSEBIS modes: 1-{nmodes}") - print(f" ℓ grid: {len(ell)} points in [{ell.min()}, {ell.max()}]") - print(f" θ grid: {n_theta} points") - - # Setup cosmology - print("\nSetting up CCL cosmology...") - cosmo, tracer = setup_cosmology_and_tracer() - - # Generate theory power spectrum - print("Generating theory C_ℓ...") - cl_ee, cl_bb = get_theory_cls(ell, cosmo, tracer) - print(f" C_ℓ(ℓ=100): {cl_ee[np.argmin(np.abs(ell - 100))]:.3e}") - print(f" C_ℓ(ℓ=1000): {cl_ee[np.argmin(np.abs(ell - 1000))]:.3e}") - - # Generate theory ξ±(θ) from C_ℓ using CCL's FFTLog - print("\nComputing theory ξ±(θ) from C_ℓ using CCL...") - xip, xim = get_theory_xipm(cosmo, ell, cl_ee, theta_integration) - print(f" ξ+(θ=10'): {xip[np.argmin(np.abs(theta_integration - 10))]:.3e}") - print(f" ξ-(θ=10'): {xim[np.argmin(np.abs(theta_integration - 10))]:.3e}") - - # Compute COSEBIS from both methods - print("\nComputing COSEBIS from harmonic-space (C_ℓ)...") - cosebis = COSEBIS(theta_min, theta_max, nmodes) - - # Fine θ grid for W_n(ℓ) filter functions - # Note: Using 20000 points for better high-ℓ accuracy (FFT-log issue) - theta_grid = np.logspace(np.log10(theta_min), np.log10(theta_max), 20000) - ce_harm, cb_harm = cosebis.cosebis_from_Cell( - ell=ell, Cell_E=cl_ee, Cell_B=cl_bb, theta=theta_grid, cache=True - ) - - print("\nComputing COSEBIS from config-space (ξ±)...") - # For config-space, need dtheta for integration - dtheta = np.gradient(theta_integration) - - ce_config, cb_config = cosebis.cosebis_from_xipm( - theta=theta_integration, dtheta=dtheta, xi_plus=xip, xi_minus=xim, cache=True - ) - - # Compare - print("\n" + "=" * 60) - print("Results: E_n comparison (harmonic vs config-space)") - print("=" * 60) - print(f"{'Mode':<6} {'Harmonic':>14} {'Config':>14} {'Ratio':>10} {'Diff (σ)':>10}") - print("-" * 60) - - for n in range(nmodes): - ratio = ce_harm[n] / ce_config[n] if ce_config[n] != 0 else np.nan - # For theory, expect exact agreement, so σ is not meaningful - # Just report absolute difference - diff = ce_harm[n] - ce_config[n] - print( - f"E_{n + 1:<4} {ce_harm[n]:>14.6e} {ce_config[n]:>14.6e} {ratio:>10.4f} {diff:>10.2e}" - ) - - print("\n" + "=" * 60) - print("Results: B_n (should be ~0)") - print("=" * 60) - print(f"{'Mode':<6} {'Harmonic':>14} {'Config':>14}") - print("-" * 60) - - for n in range(nmodes): - print(f"B_{n + 1:<4} {cb_harm[n]:>14.6e} {cb_config[n]:>14.6e}") - - # Summary statistics - print("\n" + "=" * 60) - print("Summary") - print("=" * 60) - ratios = ce_harm / ce_config - print(f"Mean E_n ratio (harm/config): {np.mean(ratios):.4f}") - print(f"Std E_n ratio: {np.std(ratios):.4f}") - print(f"Max B_n (harmonic): {np.max(np.abs(cb_harm)):.2e}") - print(f"Max B_n (config): {np.max(np.abs(cb_config)):.2e}") - - # Verdict - tolerance = 0.05 # 5% agreement expected for well-resolved integration - if np.all(np.abs(ratios - 1) < tolerance): - print( - f"\n✓ PASS: Harmonic and config-space agree within {tolerance * 100:.0f}%" - ) - else: - print(f"\n✗ FAIL: Discrepancy exceeds {tolerance * 100:.0f}%") - print(" This could indicate:") - print(" 1. Insufficient integration resolution") - print(" 2. Different ℓ/θ range coverage") - print(" 3. Bug in one of the integration methods") - - return { - "ce_harm": ce_harm, - "cb_harm": cb_harm, - "ce_config": ce_config, - "cb_config": cb_config, - "ratios": ratios, - } - - -if __name__ == "__main__": - results = main() diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 3b7eecae..890271a4 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -67,7 +67,7 @@ def _dry_run(workflow_dir, targets, *extra_snakemake_args): @requires_candide_data def test_bmodes_workflow_dry_runs(): """The paper B-mode workflow must still parse and dry-run cleanly.""" - result = _dry_run(_repo_root() / "papers/bmodes", ["all_tapestry"]) + result = _dry_run(_repo_root() / "papers/bmodes", ["paper"]) assert result.returncode == 0, result.stdout diff --git a/workflow/common.py b/workflow/common.py index 4391ddbf..9927a6f6 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -363,8 +363,7 @@ def grid_of(grids, binning): """Name of the grid a binning belongs to, compared numerically. A "300" wildcard matches a 300.0 grid value. Binnings matching no named - grid (e.g. papers/bmodes' nbins=10000 convergence check) are reporting-style - measurements. + grid are reporting-style measurements. """ key = tuple(float(binning[k]) for k in XI_KEYS) for name, grid in grids.items(): diff --git a/workflow/scripts/plotting_utils.py b/workflow/scripts/plotting_utils.py index 2e62ce7c..1bc1cd4b 100644 --- a/workflow/scripts/plotting_utils.py +++ b/workflow/scripts/plotting_utils.py @@ -1,4 +1,4 @@ -"""Shared plotting utilities for claims scripts.""" +"""Shared plotting utilities for the paper figure scripts.""" from pathlib import Path From 6c2f03a92c775d2b0626d366897e6830ec4d8fd1 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 17:50:46 +0200 Subject: [PATCH 35/83] Dockerfile: build from shapepipe:develop (#360) The im_sims tag comes from a branch that no longer exists, so the image froze at whatever ShapePipe it last built. develop is the maintained image, and it no longer ships snakemake (shapepipe 10f9c535). Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj Co-authored-by: Claude Opus 5.5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0437a929..1a4183e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Development image with more bells and whistles -FROM ghcr.io/cosmostat/shapepipe:im_sims +FROM ghcr.io/cosmostat/shapepipe:develop # liblapack-dev: cosmosis's MultiNest links -llapack, and the base image ships # only the runtime liblapack.so.3 (no dev symlink). The gsl/cfitsio/fftw3 dev From 0c0d83489a0873d9862ec7e89560e765261812c4 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 18:12:05 +0200 Subject: [PATCH 36/83] Write metacal flag columns as int32; reduce_mem never narrows integers (#354) * catalog: metacal bitmasks use FITS format K, not I or the dead FLAGS_ key NGMIX_MCAL_FLAGS carries bit 30 (shapepipe#854's absent-measurement flag) alongside the native fitter bits. Format I truncates it to int16 and silently zeroes bit 30 on write, in both the FITS and the HDF5 branch of write_shape_catalog (the HDF5 path reuses the FITS column's array). The per-type NGMIX_FLAGS_{1P,1M,2P,2M,NOSHEAR} columns had the same problem from the other direction: their format entry was keyed as "FLAGS_{suffix}", which never matches the real column name "{prefix}_FLAGS_{suffix}", so the writer's float64 default masked the dead key rather than narrowing anything. Both parameter files now key and format all six metacal bitmasks the same way, at K, so they round- trip exactly and survive JointCat's optional memory-reduction pass (which only downcasts int32/float64, not int64). NGMIX_MCAL_TYPES_FAIL stays at I: it is a count in [0, 5], not a bitmask. Adds a round-trip test parametrized over both parameter files, FITS and HDF5, and reduce_mem on/off, checking 0, bit 30 alone, and bit 30 with a native bit together. Co-Authored-By: Claude Opus 5.5 * catalog: metacal bitmasks as int32; reduce_mem never narrows integers ShapePipe sets bit 2**30 in the metacal flags for a missing measurement, which needs 32 bits. The parameter files now write NGMIX_MCAL_FLAGS and the five NGMIX_FLAGS_* columns as FITS J (int32). JointCat.dtype_out with reduce_mem narrowed every int32 column outside a keep-list to int8, silently wrapping any value above 127. It now only narrows float64 to float32 (RA/Dec excepted) and leaves integer columns alone. Co-Authored-By: Claude Opus 5.5 * catalog: justify int32 metacal flags by ngmix bits 0-15, not bit 30 The flag columns stay J: ngmix's ZERO_DOF (2**15) overflows a signed 16-bit I. Tests round-trip 0, 2**12, 2**15, a combination and all 16 bits instead of 2**30. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uemfjv9ybCwtKtprksZbVY --------- Co-authored-by: Claude Opus 5.5 --- scripts/calibration/params.py | 5 +- src/sp_validation/catalog.py | 7 ++ src/sp_validation/catalog_builders.py | 25 +++--- .../tests/test_catalog_flag_roundtrip.py | 80 +++++++++++++++++++ workflow/image_sims/params_im_sim.py | 5 +- 5 files changed, 102 insertions(+), 20 deletions(-) create mode 100644 src/sp_validation/tests/test_catalog_flag_roundtrip.py diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index ea871119..7f6d7c41 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -143,7 +143,6 @@ "NUMBER", "IMAFLAGS_ISO", "FLAGS", - "NGMIX_MCAL_FLAGS", "NGMIX_MCAL_TYPES_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", @@ -152,6 +151,8 @@ add_cols_pre_cal_format["TILE_ID"] = "A7" add_cols_pre_cal_format["NUMBER"] = "J" +# Metacal bitmasks hold ngmix flag bits 0-15; bit 15 overflows signed 16-bit I. +add_cols_pre_cal_format["NGMIX_MCAL_FLAGS"] = "J" # Create key names for metacal information prefix = "NGMIX" @@ -162,7 +163,7 @@ add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") for suffix in suffixes: - add_cols_pre_cal_format[f"FLAGS_{suffix}"] = "I" + add_cols_pre_cal_format[f"{prefix}_FLAGS_{suffix}"] = "J" # Catalog parameters diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index e39d2076..76022243 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -439,6 +439,13 @@ def write_shape_catalog( Write catalogue with galaxy shapes = shear estimates. + @sc [label:convention] metacal-flag-width + The data and image-simulation parameter files supply FITS ``J`` (int32) + for metacal bitmasks, so all ngmix flag bits (0-15; ``ZERO_DOF`` = 2**15 + overflows a signed 16-bit ``I``) survive FITS/HDF5 output; ``JointCat`` + never narrows integers. + The number of failed metacal types is a count in [0, 5], not a bitmask. + Parameters ---------- output_path : str diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index 78dc7838..b6abafc0 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -437,26 +437,19 @@ def dtype_out(self, name, dtype_in): output dtype """ - # Specify columns for which original (high-precision) format - # needs to be kept and not reduced to lower precision - cols_keep_dtype = [ - "RA", - "Dec", - "FLAGS", - "IMAFLAGS_ISO", - "NUMBER", - ] if dtype_in.kind == "U": # Transform unicode to string of equal length return np.dtype(f"S{dtype_in.itemsize // 4}") - if self._params["reduce_mem"] == False: - return dtype_in - elif name not in cols_keep_dtype: - if dtype_in.kind == "f" and dtype_in.itemsize == 8: - return np.float32 - if dtype_in.kind == "i" and dtype_in.itemsize == 4: - return np.int8 + # reduce_mem narrows float64 to float32, except coordinates. Integer + # columns (bitmasks, IDs, counts) are never narrowed. + if ( + self._params["reduce_mem"] + and dtype_in.kind == "f" + and dtype_in.itemsize == 8 + and name not in ("RA", "Dec") + ): + return np.dtype(np.float32) return dtype_in diff --git a/src/sp_validation/tests/test_catalog_flag_roundtrip.py b/src/sp_validation/tests/test_catalog_flag_roundtrip.py new file mode 100644 index 00000000..ea3cac97 --- /dev/null +++ b/src/sp_validation/tests/test_catalog_flag_roundtrip.py @@ -0,0 +1,80 @@ +"""Comprehensive catalogues preserve metacal failure bits exactly.""" + +import runpy +from pathlib import Path + +import h5py +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation.catalog import write_shape_catalog +from sp_validation.catalog_builders import JointCat + +ROOT = Path(__file__).resolve().parents[3] + + +@pytest.mark.parametrize( + "params_path", + ["scripts/calibration/params.py", "workflow/image_sims/params_im_sim.py"], + ids=["data", "image-sims"], +) +@pytest.mark.parametrize("extension", [".fits", ".hdf5"]) +@pytest.mark.parametrize("reduce_mem", [False, True]) +def test_metacal_flags_roundtrip(tmp_path, params_path, extension, reduce_mem): + """Contract metacal-flag-width: neither output nor merging loses flag bits. + + Values are ngmix flag bits: none, LM_FUNC_NOTFINITE (2**12), ZERO_DOF + (2**15), and every bit 0-15 at once. Float64 inputs match ShapePipe's final + catalogue. Bit 15 overflows signed int16, and int8 wraps everything above + 127, so the columns must stay int32. + """ + with np.printoptions(): + params = runpy.run_path(str(ROOT / params_path)) + flags = np.array([0, 2**12, 2**15, 2**15 | 2**12 | 8, 2**16 - 1], dtype=np.float64) + bit_columns = ["NGMIX_MCAL_FLAGS"] + [ + f"NGMIX_FLAGS_{suffix}" for suffix in ("NOSHEAR", "1P", "1M", "2P", "2M") + ] + columns = {name: flags for name in bit_columns} + columns["NGMIX_MCAL_TYPES_FAIL"] = np.array([0, 1, 5, 2, 5]) + assert set(columns) <= set(params["add_cols_pre_cal"]) + path = tmp_path / f"comprehensive{extension}" + write_shape_catalog( + str(path), + np.zeros(5), + np.zeros(5), + np.ones(5), + add_cols=columns, + add_cols_format=params["add_cols_pre_cal_format"], + ) + if extension == ".fits": + written = fits.getdata(path, 1) + else: + with h5py.File(path, "r") as catalog: + written = catalog["data"][:] + + builder = JointCat() + builder._params["reduce_mem"] = reduce_mem + for name, expected in columns.items(): + values = written[name] + np.testing.assert_array_equal(values, expected, err_msg=name) + if name in bit_columns: + assert values.dtype.kind == "i" and values.dtype.itemsize == 4, name + reduced = values.astype(builder.dtype_out(name, values.dtype)) + np.testing.assert_array_equal(reduced, expected, err_msg=name) + + +@pytest.mark.parametrize("reduce_mem", [False, True]) +def test_reduce_mem_never_narrows_integers(reduce_mem): + """reduce_mem narrows float64 (except RA/Dec) and leaves integers intact.""" + builder = JointCat() + builder._params["reduce_mem"] = reduce_mem + for dtype in (">i4", "i8", "f8")) == np.dtype(">f8") + expected = np.float32 if reduce_mem else np.dtype(">f8") + assert builder.dtype_out("NGMIX_G1_NOSHEAR", np.dtype(">f8")) == expected diff --git a/workflow/image_sims/params_im_sim.py b/workflow/image_sims/params_im_sim.py index 1cd395c6..b7db399e 100644 --- a/workflow/image_sims/params_im_sim.py +++ b/workflow/image_sims/params_im_sim.py @@ -148,7 +148,6 @@ for key in ( "NUMBER", "FLAGS", - "NGMIX_MCAL_FLAGS", "NGMIX_MCAL_TYPES_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", @@ -157,6 +156,8 @@ add_cols_pre_cal_format["TILE_ID"] = "A7" add_cols_pre_cal_format["NUMBER"] = "J" +# Metacal bitmasks hold ngmix flag bits 0-15; bit 15 overflows signed 16-bit I. +add_cols_pre_cal_format["NGMIX_MCAL_FLAGS"] = "J" # Create key names for metacal information prefix = "NGMIX" @@ -167,7 +168,7 @@ add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") for suffix in suffixes: - add_cols_pre_cal_format[f"FLAGS_{suffix}"] = "I" + add_cols_pre_cal_format[f"{prefix}_FLAGS_{suffix}"] = "J" # Catalog parameters From fdb7c5eb4917c833f8d8201518a57fe5aac7a7f1 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 18:38:26 +0200 Subject: [PATCH 37/83] ci(docs): install from uv.lock, not a fresh PyPI resolve (#365) The docs job ran `uv pip install '.[docs]'`, which re-resolves the whole environment and breaks on unrelated upstream releases. Sync the checkout's lock (docs extra, --frozen --inexact) into the image's venv and install the checkout --no-deps on top. PRs build docs only when docs inputs change. Claude-Session: https://claude.ai/code/session_01Uemfjv9ybCwtKtprksZbVY Co-authored-by: Claude Opus 5.5 --- .github/workflows/deploy-docs.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 3efce06b..a1878051 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -1,11 +1,11 @@ name: Build and deploy API documentation # Build the Sphinx docs inside the published image — which already carries the -# full scientific stack autodoc must import — installing the *checked-out* -# package on top so the docs reflect the code under review, not the code baked -# into the image. +# full scientific stack autodoc must import — syncing the *checked-out* uv.lock +# (plus its `docs` extra) and package on top, so the docs reflect the code under +# review and never re-resolve against PyPI. # -# pull_request → build only, as a check (no deploy) +# pull_request → build only, as a check (no deploy), when docs inputs change # push: develop → build + deploy to GitHub Pages on: push: @@ -14,6 +14,12 @@ on: pull_request: branches: - develop + paths: + - "docs/**" + - "src/**" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/deploy-docs.yml" workflow_dispatch: jobs: @@ -37,8 +43,15 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + # Exactly what uv.lock pins: --frozen never re-resolves, and --inexact + # keeps the image's packages outside this closure (the ShapePipe stack, + # the glass/workflow extras) instead of pruning them. Both land in the + # image's venv (UV_PROJECT_ENVIRONMENT). The project isn't uv-packaged, so + # the checkout goes in by hand, replacing the copy baked into the image. - name: Install documentation dependencies - run: uv pip install --no-cache-dir '.[docs]' + run: | + uv sync --frozen --inexact --no-cache --no-install-project --extra docs + uv pip install --no-cache --no-deps -e . # Builds on every event; a failing build fails the PR check. Deploy is # gated to develop pushes below. From 6faf7317b41810ac5042a9abda4e43d43a235901 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 18:38:43 +0200 Subject: [PATCH 38/83] Remove spread-model leftovers (#364) Drop the unread do_spread_model flag from the image-sim params template and describe metacal's mask argument as the pre-selection it is, with the size-based galaxy cut applied inside metacal. Claude-Session: https://claude.ai/code/session_01Uemfjv9ybCwtKtprksZbVY Co-authored-by: Claude Opus 5.5 --- src/sp_validation/calibration.py | 4 +++- workflow/image_sims/params_im_sim.py | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index e0ab212e..9eaaf42e 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -697,7 +697,9 @@ class metacal: data : input galaxy catalogue mask : array of bool - mask according to galaxy selection, e.g. spread_model + pre-selection mask, e.g. flag, magnitude and footprint cuts; the + size-based galaxy selection (``rel_size_min`` < T/Tpsf < + ``rel_size_max``) is applied here masking_type : string, optional, default='gal' masking type, one in 'gal', 'gal_mom', 'star' step : float, optional, default=0.01 diff --git a/workflow/image_sims/params_im_sim.py b/workflow/image_sims/params_im_sim.py index b7db399e..09d19bc1 100644 --- a/workflow/image_sims/params_im_sim.py +++ b/workflow/image_sims/params_im_sim.py @@ -192,9 +192,6 @@ gal_mag_bright = 15 gal_mag_faint = 30 -### Spread-model -do_spread_model = False - ### SExtractor flags to keep in addition to FLAGS=0 ### (bit-coded; list of powers of 2); ### Empty list if no flags From 2832186f17e2e7080b96f9e93164713d75861284 Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 19:22:01 +0200 Subject: [PATCH 39/83] =?UTF-8?q?workflow:=20parity-checked=20launches,=20?= =?UTF-8?q?one=20output=20root,=20one=20integration=20grid,=20machine-inde?= =?UTF-8?q?pendent=20=CE=BE=C2=B1=20(#357)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests: pin pure-E/B on exact-binning and committed ξ±; drop dead catalogue paths The synthetic pure-E/B pins passed in CI and failed on candide because TreeCorr's default bin_slop/angle_slop make ξ± follow the tree's top-level split, which varies with the jackknife patches and, through min_top, with the thread count TreeCorr takes from cpu_count(). Fixed patch centres alone leave a 4-vs-48-thread spread (reporting ξ− up to 16%); exact binning removes it (1e-12). The test measures with bin_slop = angle_slop = 0, and test_b_modes pins pure_eb_from_xi on the same ξ±, committed as tests/data/pure_eb_xi_fixture.npz. The configured-path guard skips cat_config's paths.output and directory-less calibration params.input_path values. The two LFmask entries (data gone) and the six unread covmat_file keys leave cat_config.yaml, and the slow duplicate test_catalog_paths_exist goes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * cosmo_val: pin TreeCorr's min_top so ξ± does not depend on the machine TreeCorr derives min_top, the depth of its root cells, from its thread count (max(3, ceil(log2 n)), with n from cpu_count() when unset). The root cells set which pairs bin_slop approximates, so at default slop calculate_2pcf's ξ± depended on the node: on the synthetic catalogue, 48 threads move ξ± by 0.038σ against 4. The shared treecorr_config pins min_top = 6, which is what TreeCorr derives on candide's 48- and 64-CPU nodes; the same config reaches the aperture-mass, leakage and ρ/τ correlations. test_calculate_2pcf_does_not_depend_on_thread_count measures at production binning on 4 and 48 threads (fresh Catalog each, shared patch centres) and requires agreement below 1e-6σ; it is red without the pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * b_modes: one pure-E/B kernel call; tests pin each pure-E/B fact once calculate_pure_eb_correlation (modes and jackknife) and pure_eb_covariance_mc call cosmo_numba through pure_eb_from_xi, so the transform test_b_modes pins is the one every pure-E/B product runs through. The committed ξ± fixture is now a conftest fixture (pure_eb_xi) with two users. The synthetic pure-E/B test asserts that the ξ± and edges it measures equal the fixture (agreement 2e-13), and that its modes are pure_eb_from_xi of them; this replaces its copy of the mode pins, and np.savez of what it measures regenerates the fixture. test_b_modes pins pure_eb_from_xi on the fixture, which fails loudly if cosmo_numba does not import, with no skip. A failure now names what moved: the measured ξ±, the wiring into the kernel, or the transform. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * b_modes: one pure-E/B transform per jackknife realisation The jackknife covariance function indexed pure_EB(x) once per key, so every realisation ran cosmo_numba's transform six times (55 transforms at npatch=8 where 10 suffice; values unchanged). _eb_vector concatenates the modes in _EB_KEYS order for both the jackknife and the MC covariance. The synthetic pure-E/B test counts transforms around calculate_pure_eb and is red on the per-key closure (55 > npatch + 2). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * tests: glue-section header stops enumerating which tests compare values Two tests in the section compare values (pure-E/B against committed ξ±, ξ± across TreeCorr thread counts); the header defers to each test's docstring instead of listing them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: host/image parity check, catalogue config from the checkout, one output root, one integration grid - Launch-time parity: container.image_runtime reads the image's Python minor and Snakemake version (SIF via apptainer, sandbox off disk, tags skipped); common.check_host_parity stops a launch whose host Snakemake differs, with the reinstall command. configure() and image_sims.smk call it; the README install line pins --python 3.12. - The catalogue config is the launched checkout's cosmo_val/cat_config.yaml, loaded by configure() into CATALOG_CONFIG; the paper Snakefiles no longer merge it into `config`, and covariance.smk / ecut.smk read CATALOG_CONFIG. - One output root: cv_init_params passes output_dir=COSMO_VAL, cv_runner no longer chdirs into the live checkout, and CosmologyValidation drops its COSMO_VAL environment fallback. - One integration grid (R12): the cosebis grid is gone; cv_cosebis reads the integration part and the CosmoCov g covariance on that grid (the one pure-E/B uses). An npatch=1 grid defaults to the diagonal covariance, and a binning outside the named grids takes its covariance from its own patches. - The candide profile sets jobs: 100. - workflow/tests: host-launcher DAG tests on a toy checkout (P1-P3) and the real papers on candide (P4, P5 = the container smoke test, moved here); CI runs them in a workflow-dag job. test_bmodes_workflow_dry_run.py is replaced by P4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: the image lives under ~/.cache; launch guards read the real image - container.CACHE_DIR is ~/.cache/sp_validation whatever XDG_CACHE_HOME says. A job runs the image from the path the launching host resolved, and on candide XDG_CACHE_HOME is node-local /scratch: from such a shell the launch fell back to the registry tag and skipped the parity check. SPV_CONTAINER / SPV_SANDBOX remain the overrides. - test_launch_reads_the_image_under_home: a mismatched image under a fake ~/.cache stops the launch while XDG_CACHE_HOME points elsewhere. - test_papers_resolve_on_candide asserts unconditionally that the launch read a local image (no "parity unchecked"). - test_image_sims_checks_parity_at_launch: the standalone image-sims Snakefile stops on a mismatched image. - test_assemble_resolves pins each terminal file's inputs: the reporting ξ± part with its CosmoCov ng covariance, the fiducial-binning pseudo-Cl part with its NaMaster covariance, COSEBIs, pure-E/B and ρ/τ. - The container smoke test launches as the README does (no --jobs), so the candide profile's job bound is under test. - Test docstrings drop the design-table row IDs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: a ξ± part's covariance follows its patches, in one place A grid is a binning; rule xi takes cov=patch_cov(npatch) directly, so grid_cov, _named_grid and the grids' cov key go. cv_init_params loses its unused version_list, and the cosmo_val rules pass one CV_INIT. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/README: scope the output-root sentence to papers/cosmo_val Name where the other rules write: masks under the run directory's output/masks/, papers/bmodes' figures and macros under its docs/, image sims under grids_base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: SLURM jobs start whatever the launch's XDG_CACHE_HOME; P4/P5 test what a launch does A job's Snakemake inherits the launching shell's environment (--export=ALL), and a login shell may point XDG_CACHE_HOME at node-local storage: every job then fails creating its source cache. Two halves close it, each needed (shown on n33 by replaying the captured sbatch job command with srun stubbed): - the candide profile leaves source-cache out of shared-fs-usage, so a job neither reuses the launch's cache path nor creates its own under XDG; - common.py drops XDG_CACHE_HOME from what jobs inherit, because the slurm-jobstep executor forces a shared source cache on the Snakemake it starts for each job step. test_a_job_needs_no_launch_cache covers both (each mutation turns it red). P4 (test_papers_resolve_on_candide) now passes from a shell with node-local XDG_CACHE_HOME under srun, and its bmodes case also resolves an e-cut catalogue, so both CATALOG_CONFIG readers in ecut.smk are exercised. P5 hands the smoke Snakefile the image a launch resolves (no registry pull). The host launcher and CI carry snakemake-executor-plugin-slurm, as the README install line does, so the candide profile is parsed in CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * papers/bmodes: the sweep drivers read the image from ~/.cache, as container.py does container_env.sh still followed XDG_CACHE_HOME, so from a login shell that points it at node-local storage the sweep drivers ran a missing image while spv-container and the workflow found the real one. test_sweep_drivers_run_the_resolved_image pins the shell copy of the resolution (cache and sandbox precedence) to container.resolve_image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * run_2pcf: a ξ± part carries the covariance its measurement estimated Which covariance a part carries had two homes that disagreed: rule xi passed cov=patch_cov(npatch) ("diagonal" at npatch=1), while run_2pcf's own default, reached by the CLI and papers/bmodes' run_xi_sweep, was "none". The same file name then held variances or not depending on who wrote it. run_2pcf now carries what TreeCorr estimated (gg.var_method, which calculate_2pcf sets from npatch): the jackknife covariance with patches, the shot-noise diagonal without. The cov argument, --cov, the jackknife guard, rule xi's cov param and common.patch_cov (with its test) go. Rule xi hands its wildcards to grid_of directly (xi_binning_of goes). test_xi_part_carries_the_covariance_the_measurement_estimated runs run_2pcf on the synthetic catalogue at npatch 1 and 4; the previous run_2pcf fails both cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: the launch-cache test answers the profile's apptainer check with a stub The candide profile deploys with apptainer, and Snakemake asks for the apptainer binary and its version even in a dry-run, so test_a_job_needs_no_launch_cache failed on GitHub's runners, which have no apptainer. A stub apptainer on the test's PATH answers that check; the dry-run reads nothing else from it. In a CI emulation (a clean checkout, a fresh HOME, no apptainer or SLURM on PATH, the workflow-dag command) the suite goes from 1 failed, 11 passed to 12 passed. Both mutations still turn the test red there: source-cache added back to the profile's shared-fs-usage (NotADirectoryError under the blocked XDG_CACHE_HOME) and the XDG_CACHE_HOME pop removed from common.py (the job sees the launch's cache). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: the smoke Snakefile composes workflow/ as the entry Snakefiles do P5's Snakefile never imported common, so its launch did not drop XDG_CACHE_HOME: from a login shell pointing it at node-local /scratch, the job's inner Snakemake died creating its source cache (PermissionError), which says nothing about what a real launch does. The Snakefile now imports common, resolves its image with common.resolve_container and checks host/image parity, as the entry Snakefiles do. P5 launches exactly as the README does, with no --config container=, and the literal default image and the test that kept it in step with CONTAINER_URI go. Checked on n33 with sbatch stubbed to record what it would submit, then the recorded job replayed with srun stubbed, from a shell with XDG_CACHE_HOME set to /scratch/cdaley/tmp/xdg: at the parent commit the job environment carries XDG_CACHE_HOME and the replay fails with PermissionError; with this commit it carries none, the job runs in the SIF, and P5's assertions pass on its report. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: resolve_container checks the image it returns The host/image parity check had a home beside the resolver, so configure() resolved the image once only to check it, and every Snakefile that picks its own image (image_sims.smk, the smoke Snakefile) had to remember a second line. resolve_container now runs check_host_parity on the image it returns: the image checked is the image that runs, and the extra call sites go. check_host_parity stays cached, since composed Snakefiles evaluate container: more than once. Removing the check from resolve_container turns 5 host tests red (both parity cases, the unreadable-image line, the image under ~/.cache, image sims). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: P5 checks that the job imports the launched checkout's src The smoke Snakefile imported common but never put the checkout's src/ on the job's PYTHONPATH, so its job imported the image's baked sp_validation, and P5's assertion (any editable src/ layout) passed on it. The smoke Snakefile now calls inject_checkout_pythonpath as configure() does for the entry Snakefiles, and P5 asserts the job's sp_validation is this checkout's src/sp_validation/__init__.py. Run on n08 through the default profile (apptainer, no SLURM): the job reports /src/sp_validation/__init__.py; with the injection removed it reports /sp_validation/src/sp_validation/__init__.py, which the new assertion rejects and the old one accepted. P5 itself needs a SLURM submit host and has not been run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: COSMO_VAL defaults to the launched checkout's own output tree A launch from any checkout that named no COSMO_VAL read that checkout's catalogue config and code but wrote every cosmo_val product into the production tree, silently. COSMO_VAL now defaults to the launched checkout's (gitignored) cosmo_val/output, so production is written only from the production checkout or when a launch names it; COSMO_INFERENCE keeps its shared default. README says so. P2 gains an unnamed case: with COSMO_VAL unset, every declared output lies under the toy checkout's cosmo_val/output, the inference root or results/. Restoring the production default turns it red (outputs under /n17data/cdaley/unions/code/sp_validation/cosmo_val/output). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * calculate_2pcf: the ξ± text dump carries columns only, so it reads back With patches, calculate_2pcf wrote its .txt with write_cov=True and no per-patch results. TreeCorr 5.1.4 writes num_rows only alongside patch results, so its reader ran on into the cov block and raised ("got 12 columns instead of 11"). The two ξ± figure rules re-enter calculate_2pcf on the reporting grid (npatch=100) and hit exactly that read. The dump now carries the columns only; the covariance matrix lives in the SACC part, and the figure readers use the columns. test_a_patched_xi_dump_reads_back measures at npatch=4, then re-enters calculate_2pcf from a fresh CosmologyValidation and compares the columns; under write_cov=True it fails with the ValueError above. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: the candide profile's job bound is checked on any host A real launch through the committed candide profile, of an up-to-date target, submits nothing, so it runs in CI and on an allocation; the same launch through the profile without `jobs` is refused. test_container_smoke submits a real job and needs sbatch, which only a login node has; it is documented as the login-node check it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * profiles: wait 60 s for a job's outputs to appear P5 (one real SLURM job through the candide profile, from a login node) passed, but only on its retry: the job finished, its output took more than 5 s to show on the login node's /home, and Snakemake re-ran it. On a multi-hour job that retry costs hours, and a second miss fails the run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * rho_tau_stats: 6 h wall clock; the jackknife ρ/τ outruns the 60 min default Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * workflow: no snakemake in the image, so any host Snakemake works A script: job unpickles the host's snakemake object with whichever snakemake it imports first, and the image's site-packages precede the host's appended sys.path. The image carried its own (the workflow extra, and the base image's jupyter extra), so the host had to match its version exactly. The Dockerfile now uninstalls every snakemake* package after the sync and the workflow extra drops snakemake; the job then reads the pickle with the package that wrote it. The launch check keeps only the Python minor (check_host_python): the host's snakemake and its compiled dependencies load into the image's interpreter. The README install line, CI's DAG job and the test harness no longer pin a Snakemake version. The container smoke job now reports which snakemake unpickled its object and asserts it is the host's version. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * profiles: every key needed and non-default, each with its reason Drops the candide LD_LIBRARY_PATH to /softs/openmpi: /softs is not bound, so the path does not exist inside the container, and no containerized rule uses MPI. Drops the default profile's --bind /home (apptainer mounts $HOME already). States the real reasons for the rest: rerun-triggers leaves out software-env because it hashes the per-person image path; shared-fs-usage leaves out source-cache because jobs would be handed the launch's node-local cache path; slurm_account because the executor's guess fails on candide; retries and kept logs for fan-outs and their printed output. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * workflow: jobs read the repo's matplotlibrc, never the launching user's Apptainer binds $HOME, so a job read the launcher's own matplotlibrc, and a LaTeX preamble there the image cannot typeset stopped every figure rule. common.py points each job's MATPLOTLIBRC (through APPTAINERENV_, past --cleanenv) at an empty workflow/matplotlibrc. The container smoke job reports the matplotlibrc it would read and the test asserts it is the repo's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * CONTRACTS: the host-side workflow imports only stdlib and snakemake workflow.common runs in the host Snakemake with no sp_validation installed, and loads container.py by path; container.py imports only the standard library, since it also runs as the spv-container CLI before any image exists. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * profiles/default: bind /home; Snakemake's --home keeps apptainer from mounting it A job's home is its working directory, so a checkout or output tree under the launching user's home was invisible to the job: the toy run's jobs imported the image's sp_validation instead of the checkout's src/. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * workflow: jobs read the user's matplotlibrc; no snakemake-uninstall step The image's TeX now carries sfmath, so a usetex preamble typesets inside jobs and the repo matplotlibrc override has nothing left to guard. The shapepipe:develop base ships no snakemake, so the uninstall step is gone; pyproject says why the workflow extra must not re-add it. Co-Authored-By: Claude Opus 5.5 * workflow: resolve the image without a host/image Python check; trim profile and README The launch resolves its image and runs it; the README states that the host Snakemake runs on the image's Python (3.12). The CONTRACTS notes go (the workflow-dag CI job is what enforces host-importability). The candide profile excludes n17 and n36 only, and the README's output-roots paragraph and the profile's shared-fs-usage comment say what they need to in fewer lines. The DAG tests drop the Python-check tests and the profile job-bound test, and test_one_integration_grid checks only the shared COSEBIs/pure-E/B inputs. Co-Authored-By: Claude Opus 5.5 * tests: drop the smoke test's numeric check and perf-pinning asserts The container smoke job reports and checks placement, imports, Snakemake version and provenance; the host-side suite no longer needs numpy. test_xi_grids loses its grid-count test, and the pure-E/B cosmo_val test keeps its value checks without counting kernel calls. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/deploy-image.yml | 20 + Dockerfile | 5 +- cosmo_val/cat_config.yaml | 88 ----- papers/bmodes/Snakefile | 1 - papers/cosmo_val/Snakefile | 1 - papers/cosmo_val/config/config.yaml | 8 +- pyproject.toml | 4 +- src/sp_validation/b_modes.py | 57 ++- src/sp_validation/container.py | 5 +- src/sp_validation/cosmo_val/core.py | 13 +- src/sp_validation/cosmo_val/real_space.py | 12 +- src/sp_validation/tests/conftest.py | 19 + .../tests/data/container_smoke/Snakefile | 21 - .../tests/data/pure_eb_xi_fixture.npz | Bin 0 -> 16564 bytes src/sp_validation/tests/test_b_modes.py | 77 +++- .../tests/test_bmodes_workflow_dry_run.py | 88 ----- .../tests/test_config_paths_exist.py | 19 + .../tests/test_container_smoke.py | 126 ------ src/sp_validation/tests/test_cosmo_val.py | 331 +++++++--------- .../tests/test_cv_init_params.py | 1 - src/sp_validation/tests/test_xi_grids.py | 26 +- uv.lock | 369 ------------------ workflow/README.md | 55 ++- workflow/common.py | 91 ++--- workflow/profiles/candide/config.yaml | 84 ++-- workflow/profiles/default/config.yaml | 46 +-- workflow/rules/cosmo_val.smk | 113 +++--- workflow/rules/covariance.smk | 4 +- workflow/rules/twopoint.smk | 11 +- workflow/scripts/cv_cosebis.py | 16 +- workflow/scripts/cv_runner.py | 8 +- workflow/scripts/cv_summarize_bmodes.py | 2 +- workflow/scripts/run_2pcf.py | 29 +- workflow/tests/conftest.py | 222 +++++++++++ workflow/tests/data/container_smoke/Snakefile | 27 ++ .../data/container_smoke/container_smoke.py | 41 +- workflow/tests/pytest.ini | 3 + workflow/tests/test_container_smoke.py | 90 +++++ workflow/tests/test_dag.py | 173 ++++++++ 39 files changed, 1066 insertions(+), 1240 deletions(-) create mode 100644 src/sp_validation/tests/conftest.py delete mode 100644 src/sp_validation/tests/data/container_smoke/Snakefile create mode 100644 src/sp_validation/tests/data/pure_eb_xi_fixture.npz delete mode 100644 src/sp_validation/tests/test_bmodes_workflow_dry_run.py delete mode 100644 src/sp_validation/tests/test_container_smoke.py create mode 100644 workflow/tests/conftest.py create mode 100644 workflow/tests/data/container_smoke/Snakefile rename {src/sp_validation => workflow}/tests/data/container_smoke/container_smoke.py (61%) create mode 100644 workflow/tests/pytest.ini create mode 100644 workflow/tests/test_container_smoke.py create mode 100644 workflow/tests/test_dag.py diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index f65ab5f7..19995403 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -9,6 +9,26 @@ env: BRANCH: ${{ github.ref }} jobs: + # The workflow's DAG properties, checked through the host launcher: Snakemake + # on the image's Python with sp_validation absent, as on a user's machine. + workflow-dag: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + + - name: DAG tests + run: >- + uv run --isolated --no-project --python 3.12 + --with snakemake --with snakemake-executor-plugin-slurm + --with pytest + pytest workflow/tests -m "not candide" + build-and-push-image: runs-on: - ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 1a4183e0..f9272fd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,9 +72,8 @@ WORKDIR /sp_validation # our lock — instead of pruning them. Copy the lock + manifest first so this # layer caches independently of source edits. Extras: test (CI unit suite), # glass (GLASS map-level mock — pulls glass.ext.camb + the cosmology wrapper), -# workflow (Snakemake + mpi4py runners). cs_util 0.2.2 (with cs_util.size) and a -# numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake -# and cs_util `--upgrade` layers are gone. +# workflow (mpi4py, CosmoSIS and the other rule-script runners). cs_util and a +# numba-safe numpy come straight from the lock. COPY pyproject.toml uv.lock /sp_validation/ # cosmosis builds MPI-enabled polychord/multinest only when MPIFC is set: its diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index a4d34bee..218ba413 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -914,7 +914,6 @@ SP_v1.4.11.3: e2_star_col: HSM_G2_STAR shear: R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des @@ -960,7 +959,6 @@ SP_v1.4.12.3: e2_star_col: E2_STAR_HSM shear: R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des @@ -1006,7 +1004,6 @@ SP_v1.4.13.3: e2_star_col: E2_STAR_HSM shear: R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des @@ -1072,7 +1069,6 @@ SP_v1.4.6.3_uncal: subdir: /n17data/UNIONS/WL/v1.4.x shear: path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1088,7 +1084,6 @@ SP_v1.4.6.3_uncal_w_iv: subdir: /n17data/UNIONS/WL/v1.4.x shear: path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1104,7 +1099,6 @@ SP_v1.4.6.3_uncal_w_1: subdir: /n17data/UNIONS/WL/v1.4.x shear: path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1160,88 +1154,6 @@ SP_v1.4.8_uncal: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 -SP_v1.4_LFmask_8k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: black - getdist_colour: 0.0, 0.5, 1.0 - ls: solid - marker: d - cov_th: - A: 2137.7977618140676 - n_e: 7.976462506484096 - n_psf: 0.5434016250405327 - sigma_e: 0.31509572849714534 - psf: - PSF_flag: HSM_FLAG_PSF - PSF_size: HSM_T_PSF - star_flag: HSM_FLAG_STAR - star_size: HSM_T_STAR - hdu: 1 - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: HSM_G1_PSF - e1_star_col: HSM_G1_STAR - e2_PSF_col: HSM_G2_PSF - e2_star_col: HSM_G2_STAR - label: SP_LFmask_psf - shear: - R: 1.0 - path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits - w_col: w - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - patch_number: 150 -SP_v1.4_LFmask_8k_noalpha: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: brown - getdist_colour: 0.0, 0.5, 1.0 - ls: solid - marker: d - cov_th: - A: 2137.7977618140676 - n_e: 7.976462506484096 - n_psf: 0.5434016250405327 - sigma_e: 0.31509572849714534 - psf: - PSF_flag: HSM_FLAG_PSF - PSF_size: HSM_T_PSF - star_flag: HSM_FLAG_STAR - star_size: HSM_T_STAR - hdu: 1 - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: HSM_G1_PSF - e1_star_col: HSM_G1_STAR - e2_PSF_col: HSM_G2_PSF - e2_star_col: HSM_G2_STAR - label: SP_LFmask_psf - shear: - R: 1.0 - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits - w_col: w - e1_col: e1_cor - e1_PSF_col: e1_PSF - e2_col: e2_cor - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - patch_number: 150 nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz dndz: diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index 6af554bb..144097d2 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -2,7 +2,6 @@ # composed over the generic compute workflow at ../../workflow/. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/papers/cosmo_val/Snakefile b/papers/cosmo_val/Snakefile index 67bef0a7..083dd2a6 100644 --- a/papers/cosmo_val/Snakefile +++ b/papers/cosmo_val/Snakefile @@ -7,7 +7,6 @@ # isolation or as the whole suite via the default `cosmo_val_all` target. configfile: "config/config.yaml" -configfile: "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" envvars: "PYTHONUNBUFFERED", diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index ae99aba4..5d8416c6 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -74,18 +74,14 @@ cosmo_val: kmax: 20 kmax_extrapolate: 500 - # The fine ξ± grid the B-mode integrals run over. + # The fine ξ± grid both B-mode statistics (COSEBIs, pure-E/B) integrate over. integration: min_sep: 0.08 max_sep: 300 nbins: 1000 - # COSEBIs decomposition (config space, fine integration binning) + # COSEBIs on the integration grid: mode count and the scale cuts scanned. cosebis: - min_sep_int: 0.9 - max_sep_int: 300 - nbins_int: 1000 - npatch: 100 nmodes: 20 scale_cuts: [ [1, 250], [2, 250], [5, 250], [10, 250], diff --git a/pyproject.toml b/pyproject.toml index 632793f4..a8335f10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,9 +157,9 @@ glass = [ ] # Cosmo-inference workflow runners (workflow/scripts/*). Kept optional: the core # library resolves without them, but the container installs this extra so the -# Snakemake workflow and cross-validation runners are available. +# workflow's rule scripts and cross-validation runners are available. No +# snakemake: a `script:` job must unpickle with the host's (workflow/README.md). workflow = [ - "snakemake", # Optional MPI runners (the container ships OpenMPI at /opt/ompi, so mpi4py # builds against it). "mpi4py", diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 7a3cd484..fdc2ddf8 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -19,6 +19,11 @@ _EB_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") +def _eb_vector(modes): + """Pure-E/B modes concatenated in ``_EB_KEYS`` order, the covariance layout.""" + return np.concatenate([modes[k] for k in _EB_KEYS]) + + def find_conservative_scale_cut_key(results, requested_scale_cut): """ Find scale cut key that conservatively fits within requested range. @@ -156,23 +161,20 @@ def calculate_pure_eb_correlation( dict Dictionary containing pure E/B mode results and covariance """ - from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes - # Calculate min_sep and max_sep from gg object min_sep, max_sep = gg.left_edges[0], gg.right_edges[-1] def pure_EB(corrs): gg, gg_int = corrs - return get_pure_EB_modes( - theta=gg.meanr, - xip=gg.xip, - xim=gg.xim, + return pure_eb_from_xi( + theta_report=gg.meanr, + xip_report=gg.xip, + xim_report=gg.xim, theta_int=gg_int.meanr, xip_int=gg_int.xip, xim_int=gg_int.xim, tmin=min_sep, tmax=max_sep, - parallel=True, ) # The results dict is self-describing: the grids it was measured on travel @@ -190,7 +192,7 @@ def pure_EB(corrs): "xim_int": gg_int.xim, "n_eff": n_samples if cov_path_int is not None else gg.npatch1, } - results.update(dict(zip(_EB_KEYS, pure_EB([gg, gg_int])))) + results.update(pure_EB([gg, gg_int])) if cov_path_int is not None: if z_dist is None or cosmo_cov is None: @@ -214,7 +216,7 @@ def pure_EB(corrs): results["cov"] = treecorr.estimate_multi_cov( [gg, gg_int], var_method, - func=lambda x: np.hstack(pure_EB(x)), + func=lambda x: _eb_vector(pure_EB(x)), cross_patch_weight="match" if var_method == "jackknife" else None, ) @@ -236,8 +238,7 @@ def pure_eb_from_xi( ): """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. - The values-only seam of :func:`calculate_pure_eb_correlation`, for callers - holding ξ± arrays rather than TreeCorr correlations. + The one place this module calls cosmo_numba's Schneider (2022) transform. ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* (``gg.left_edges[0]`` / ``gg.right_edges[-1]``). The reporting grid must be a @@ -281,15 +282,13 @@ def pure_eb_covariance_mc( ξ± draws come from ``cov_int``, a ξ± covariance on the integration grid, around the theory mean for ``(z, nz)`` under ``cosmo``; each draw is binned - down to the reporting grid and pushed through ``get_pure_EB_modes``. The + down to the reporting grid and pushed through :func:`pure_eb_from_xi`. The covariance of the transformed draws is the result, so it depends on the covariance model and the grids, never on the measured data vector. Returns ``(cov, eb_samples)`` — the covariance in ``_EB_KEYS`` order and the draws behind it. """ - from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes - theta, theta_int = np.asarray(theta), np.asarray(theta_int) nbins_int = len(theta_int) @@ -318,23 +317,21 @@ def pure_eb_covariance_mc( samples_rep_xip = (binning_matrix @ samples_int_xip.T).T samples_rep_xim = (binning_matrix @ samples_int_xim.T).T + def eb_draw(i): + modes = pure_eb_from_xi( + theta_report=theta, + xip_report=samples_rep_xip[i], + xim_report=samples_rep_xim[i], + theta_int=theta_int, + xip_int=samples_int_xip[i], + xim_int=samples_int_xim[i], + tmin=left_edges[0], + tmax=right_edges[-1], + ) + return _eb_vector(modes) + eb_samples = np.array( - [ - np.concatenate( - get_pure_EB_modes( - theta=theta, - theta_int=theta_int, - xip=samples_rep_xip[i], - xim=samples_rep_xim[i], - xip_int=samples_int_xip[i], - xim_int=samples_int_xim[i], - tmin=left_edges[0], - tmax=right_edges[-1], - parallel=True, - ) - ) - for i in tqdm.tqdm(range(n_samples), desc="MC samples") - ] + [eb_draw(i) for i in tqdm.tqdm(range(n_samples), desc="MC samples")] ) return np.cov(eb_samples.T), eb_samples diff --git a/src/sp_validation/container.py b/src/sp_validation/container.py index 7aa5f2ed..0bc58e11 100755 --- a/src/sp_validation/container.py +++ b/src/sp_validation/container.py @@ -46,7 +46,10 @@ # per branch, sanitized, so ``:develop`` tracks the integration branch. CONTAINER_URI = "docker://ghcr.io/cosmostat/sp_validation:develop" -CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "sp_validation" +# Under the home directory, which every node mounts: a job runs the image from +# the path the launching host resolved, so the image cannot sit on node-local +# storage -- where a cluster's XDG_CACHE_HOME often points. +CACHE_DIR = Path("~/.cache/sp_validation") # Where this user's image lives. Per-user by construction: one file, one owner, # no coordination. Override with ``SPV_CONTAINER`` (an absolute path). diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 7da61279..1ea75134 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -83,8 +83,7 @@ class CosmologyValidation( Path to catalog configuration YAML defining survey metadata, file paths, and analysis settings for each version. output_dir : str, optional - Override for output directory. If None, falls back to the COSMO_VAL - environment variable, then to the catalog config's paths.output. + Output directory. If None, the catalog config's paths.output. rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' Fitting method for PSF leakage systematics parameters. cov_estimate_method : {'th', 'jk'}, default 'th' @@ -313,6 +312,11 @@ def __init__( "nbins": nbins, "var_method": var_method, "cross_patch_weight": "match" if var_method == "jackknife" else "simple", + # min_top sets the depth of TreeCorr's root cells, hence which pairs + # bin_slop approximates. Left unset, TreeCorr derives it from its + # thread count (max(3, ceil(log2 n))) and ξ± depends on the machine. + # 6 is what TreeCorr derives on candide's 48- and 64-CPU nodes. + "min_top": 6, } self.catalog_config_path = Path(catalog_config) @@ -391,11 +395,6 @@ def ensure_version_exists(ver): self.versions = final_versions - # Override output directory: explicit arg > COSMO_VAL env var > - # catalog config's paths.output. The env hook lets a reproduction - # run redirect every product to a fresh tree without touching the - # per-script call sites (mirrors workflow/common.py's COSMO_VAL). - output_dir = output_dir or os.environ.get("COSMO_VAL") if output_dir is not None: cc["paths"]["output"] = output_dir diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 1521ecc7..89dad2a9 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -94,13 +94,11 @@ def calculate_2pcf(self, ver, npatch=None, **treecorr_config): # Process the catalog & write the correlation functions gg.process(cat_gal) - # Never write_patch_results: a per-patch ξ± realisation is an - # unblinded data vector, and nothing downstream reads one — the - # covariance a consumer needs is the matrix, which the SACC part - # carries. The .txt keeps the matrix only where there are patches to - # estimate it from; at npatch=1 var_method is "shot" and it would add - # nothing over the varxip/varxim columns. - gg.write(out_fname, write_patch_results=False, write_cov=int(npatch) > 1) + # Columns only. The covariance matrix lives in the SACC part; a + # per-patch ξ± realisation is an unblinded data vector nothing reads; + # and TreeCorr cannot read back a text file carrying the matrix + # without the per-patch results. + gg.write(out_fname, write_patch_results=False, write_cov=False) # Add correlation object to class if not hasattr(self, "cat_ggs"): diff --git a/src/sp_validation/tests/conftest.py b/src/sp_validation/tests/conftest.py new file mode 100644 index 00000000..40424120 --- /dev/null +++ b/src/sp_validation/tests/conftest.py @@ -0,0 +1,19 @@ +"""Fixtures shared across test modules.""" + +from pathlib import Path + +import numpy as np +import pytest + +PURE_EB_XI = Path(__file__).parent / "data" / "pure_eb_xi_fixture.npz" + + +@pytest.fixture +def pure_eb_xi(): + """Committed ξ± of the synthetic coherent-shear catalogue. + + Exact-binning reporting [15, 70]′ in 6 bins and integration [1, 300]′ in + 600 bins, keyed by ``b_modes.pure_eb_from_xi``'s parameters. + """ + with np.load(PURE_EB_XI) as npz: + return {k: (v.item() if v.ndim == 0 else v) for k, v in npz.items()} diff --git a/src/sp_validation/tests/data/container_smoke/Snakefile b/src/sp_validation/tests/data/container_smoke/Snakefile deleted file mode 100644 index 75527690..00000000 --- a/src/sp_validation/tests/data/container_smoke/Snakefile +++ /dev/null @@ -1,21 +0,0 @@ -# Standalone workflow exercised by src/sp_validation/tests/test_container_smoke.py. -# -# The module-level `container:` below mirrors what every real workflow does -# (workflow/Snakefile) -- Snakemake has no way to take a default image from a -# profile. Everything else under test arrives from the driving profile -# (workflow/profiles/candide). See container_smoke.py for what the job checks. - - -# Literal rather than the package's CONTAINER_URI: this Snakefile is test data, -# deliberately standalone. test_container_smoke.py asserts the two agree, so -# drift fails the test rather than the run. -container: config.get("container", "docker://ghcr.io/cosmostat/sp_validation:develop") - - -rule container_smoke: - output: - "results/container_smoke.yaml", - resources: - runtime=5, - script: - "container_smoke.py" diff --git a/src/sp_validation/tests/data/pure_eb_xi_fixture.npz b/src/sp_validation/tests/data/pure_eb_xi_fixture.npz new file mode 100644 index 0000000000000000000000000000000000000000..fa67b006d5fae626006bc7670bcafba9e7558242 GIT binary patch literal 16564 zcmd6OXH*r-x-B{9oLS_YbLy|iIU|T58384W7zm016%Y)dhyep)LIgoUlvPSpK@b!a zPy|sxlH?!)kA2VCZ@hiReZTIX+iUdtM)&Hus%F>M)m5{5y4X|Gs8IZCk)ZhYu)}`i z-wOu?4@E+FXhNWWd}wS;e1d9p?0!lL4vMmWAN_Yc=d{Jq#ha2Mks?VhBy@LhyqqCP z&UlBuoH9vn$0ja5Fxo#RJ|y(7{Z@gIyF)kk?+y=)4c*+9*Hu3mh!LW+WVRV56TSe9wPH&NbHo5h-Vk`nzQ0r5j zT0!_HU#$wtrhoE{`d`jhLa)o`?d@fPGg2^p^wMMUg+0FrLwG@6`W#Ck|K&9~#o(7+ z*xg|=mwkD9@YG%6pM0~MM``}%%ldyaIwJc2rZeLHS3dt8|7Yo}uKqtvXeMV{BvNpU4g#WbPkXUMStVYIe4r6 zti<5P9LVXHUf1lJgPR>rW!I?Yp;7l?>3f}dxR}U0{xWDD>`h!o+Op^2OX*%&y_R`! zS2j?sn4Jff*Mac;9((k^$_(UJW2g-?x48{ zx0)FxJ9jO@z%An)@x_ZEwv;qK{b4f}hn3FoEy3fUT}e|`OOVJ`wjp+W2^5U7Z?@Jh z!S83pZl0AT7(KeZwnk$a&iD)$<%ciB6x-;C+}&mHABm>w9a)AD?h686#aBQh(rLmr zcm+6bZN0Yl(h5)%4jvBcS%IPXqe5ryp?YhblAx8}IuI#_hoEhdGmLn7q^_2CEW z;FA5+^VFYpIJAv(i-gSvppc@pmYfaf4j>sk|GWV$E+=()1u2lvkjXTgF$Jmy9N`xa4!aTGn(b3K-mp|T==xWr{ zr@|*Fk>9f=e%eY(R21Xt_~bJs;*BUCOk|`&EWiAz<#njgZN8xLRX-}k_vno8R0V`JyTZKzTAh5fAFd#RCL#8(uBMUwNtP zM{3mTt|wVUPlMiH`o2)0OM^lV+KpV?NrRFbb5sspq(R%ewA=T*p+S;=R`wfi&>+^^ zR;FjQXpvs3V|i#8Ei$dWb-*T{7X5to*irL6Evi10U&%*9hs5hkbvupekeq$B1&*ad zez$l+s_)Vvwnt(8-CyYt-#+5E7au(mkPff1aHdBd+Qwh|kI^Gfm%4@Q7xbvD^^$qw zZ+cYoL&R5EmjNaHZY~~ zm6r+`QE|HZ6#Eb(nx#dj?#ii;ink-1rET9} zJue*{n0ZiML!OzeArCrrzuRMa7Y{0||0vddl?O>sXH96o<3XzzwhabQ^P#IvQJOHQ5J0&c4>QxI1<+JdTUW5MAWGC0p|OY%M6c-<Z`CL6QV8h%D2sx^L%`8ZngX@A0UUP8`iD zbRJE07e^wDmwcZTilexLICjgVIQsa#WWh|7gzje0sK2u$p_={Q)NeN z@-p5*#ZM-oTiu->ltxHM-+D2754!|P>@DV2F_1t3#@@yQArh#UvF=bvwgd{!cr?8A zsRUZ%ZuUjrB~W3MmHJ0^N#v+K_uJT568&m8U2!;E66GGbcWp3F5(R{u*j8RIiQ4UR ztAl1F5sjDe)E+S@Btp-{^U_fYeW=Q>IFTxaz8UbzM3+b*g+lh0>d#WhFSG9416FDD zw!1-V2S_86Ju!WJanfigr#VadhBSI2xN%(SgESf!Y!KjMkU?G?I`le*GKhFl)ano; zgYviiaG$;@gC5F995L^aLF8qaw_=k;ojfB$aptlp${R*V92vbEk9S)zb!b}3C?9alg{ zuTKYjS5id%ug{qG?NUUAKRl@=N)*x8AT|^*sfbXgIEhmoN2>A0~nQb?Djk24S~4# z zU7XoNF{X|z{7-jgb8Dc2cjSkrCK~7ubByN6Fb$;c#VWLLMgtjMiR(hO8fb35#M^6B z16|V?H|gWnMD)}K@fVRM(mtcDJ`|~mR-f7q@4l#s+KuRcb2n=uEm!lr_=U|V{q%B& zloq11r@egAO$$v(bwr*`(?V;OzVqhgT4=BBLRa{p7E&FNZ;<8EM$*Oyc59kzqY{OX z=r;-4=)l_gAopfp++4bK4@-|WvNOlemJqno}hyQay>F#?`-aW zZECXZiw>Fz@#fjXrHd4ltwot^bP@kowvPE!UGy_vmB;0YF3M8e8pb@Si-IB@1!QFO z(534^`=+<+p%~gTKNT+MA&-$@?eq>k^dr6Xlqs`5@?#tgG_lf0*2nL>xtykta-P&m z3e@VOUM}wn#ozkq#C9!}MqLAR%E|MzNP+>Pk!9~Gcw~UmP0-Mp83W|9Euvaj!w@~+ zLlhas8KSp>R);>78zMIS14(pqhG>!IZNznLBgAi~B))I25xU`qg6L|D(1^@{?b4bN z!fm=$!sfJ`lJ|@U+Cr{n28zw07gl$jm zxCsi)Pn2lTCXgNxE#Z+$pf@fFA@~)6j6YvZ*~n|bHW850CwE-b!>-VdflL+l!Nsrg! zH$@jNCg&s~Q)J~n;PHN^DYAUZX?7yd6fFhnJ#~1o8SUoIH;$X4nQt@))A`NNw0qt* zb_+8kSP*wAE!GT$DQJgXykdqPlOn}TTFekM%XJ=GF++OK?sg=|nWOXbY*R{}=19nB zP3Yb+bHwaNpR((*Il5TSyS?g%If{P9IxNj+f!6PG@x@tNpbyFsECKs1km$vyhqm0e zK+Xk;X<1(^5L4vL{CN&bG%sSWB4S~QzJ`9k$G^`K@z7++<(F8ZjZ-I3$)F`VzSqo{ zhTjUMvs>%aIa;9$HBBrhj##0CU1NofHC8B28qq}W-;n39L5e`{GqCl!EiFW8GeHRr+ zl^qIIIm8>WY=`hgZ{Z?Cd(@PAEx{3>Bo;;Efbw6jh_u=`pt7=< zexYm!BzbgE@BMoR^zP2qFH}N~NHF_qVOXFeveU49A$`{oP0}QANX5c+idd3Q$x})XBTEmH-?&$R6?8rM1F4}BCmHLj}_ma1u%sF^fh3CFI#7xqN- z5utA5uAZoe>xN3!2~VV2R=8AyJ<+T1J3aPGo~SD7RcO4L7ji4QrbZp^h5qarjSjx< zg_f?`eI4!fLOPQfDek;mkX>y|}Zz;Ey+|F5T}yGTw$hJZAyrhLLkeH;3nULhESe2{1Ju3p9?KBy_+4$IaiA9PeDTrY*m7s;`g@1A$^MbiyCkICoy zqI6PG(?X9gx^RV-%bv6y9ViZ@d>OVKP295C-SKcc3g=SFHDBJ26r4ncPMi9n;Vuid zf=oYzj@)#M>-Iy3nOIEDN%$jG6QVXZ${#V3oPyhG{Lz;O9`}W40?@c#a;vja06M2} zHKD~V04eKM4%Hk8K+=9Haa=b8&X0*9CnXCZ!Qp>zSwv(O*{xGoLP09un$57yG&Oz z_64DYjGJ4@w}OxcTs-dlJ_v=Y*-Qjd2BZ62{Z1`v!RWd1@2>ZL!N^EDev&^s8112> zr5UXaM&E-!Nm)(>qxp^3?E_*VNTh2&7vU0uI9u*oET@H_WXR59t_(rJRFl_AhC@(? z0wZO;U?}R)`Bk{nF%$(*_eGyg3q=JxTc|~!gd$(IjAiBVP_({a_-2T-10@AH)irtU zKxBFkn(R|M&2+}!VsK`- zdUhZT<=h@+`XCgJN|(-+f7lX^1bn{59Xu0`K0d6si+UT5@S$#7Ev5+cHesdB);aLZZ#;dAQ8sdl2*H$u+nnC(PQ=)x^c4(~+mQ&lF8^*a%tY4z?x znq8>$i078)R=d#koU-YyCwC$7M3?B^)?H|R`S{Knj!1OTD$HDBOC*Y<4eKl_h(sGf z1yzRzA`v~O;E6x7QHXMdHOnqK3h^DDv)=wB3T5|I-`rS_Li#Cmg$g#&NPOL6Z^_wc z^RT^ED)Z>L;gV2DK>cP|^Ssm7vN8Wpv4 z&selkr(1S8H5S=MZkP-e$D$)|`=~nGW6|{P^L#nWv4}?flE+rbI5g2ZaZtq}4xQFD zq>fCCL&>*C7=y0Ip@c9l-^A89L=z*U@?8P*_f`G_>_2* z_0-_A69`v=}4i1d9FB3m_L#xzV z(~?FXh?ToL#bcueM`q0*lr3oBg3=4OKOS)hjpeaNV_vFwDbH6YL(c$&U!HsVs>>2n zXjUaZP&R>wR>cgoBUU(~Z|YNboiDy^KQq`+>jFy06saG6(#2`7z5i?p3W6_bBTFAI zo8a1AvK60PY;aO%>|;M&FE~>-qg;O%VM7bODC0Y}Ac*%nOiuWKbq()Q@wg>+8x%~h z@Nfhk#gQ-HBRpU^a6A1iL3i-i=r0@ybB9Jje?NyfL9D7t*68fqf+>xco{K6=;zzDD zS~oczL9E1!Qpj5tGuLQNnW$T1)2I4P>E=4H^z9qNjW`{APb5u)dY=YV3QaZCIdbDi zJl~k3oi*|7gn35kxC3~{Py0B``hw}s(gYHZ0WgyEr%NRTaU;jx54h6}E@T#19QmvU zC7t>|oGC1@%Po%2Pt#1H-|AUPRh2F5()soI9@4_BD-9DY=Mm)X2 zHg1+>9a&S*s!cpA{YMvThPbiBxR_(h8ylZ=ln{_O@=^|;7R7?*bfak;+E4Y^(smmF}14;t5`R4n0AmY+}F`FF}IP6M>5JL(hK3#iCeJDo( z&c*BTxZSY;)fBI`kpsF|FqiH4;a8UUcs1R^c5PcsePZU8Ww<_c(PYjhKQje&4JL<4 zvN344HxGaEwt?y9v)7L1Xv0|*#oA+^=7_6t^qs<61%cPAP-5w*EEX>BIohv0LsTS3 z#UIGE1;uWCo}42F5I!W&9@ndZFR&#)efdZk&lS@xdEErOC}f<9ESO>C z)T!@&lX_rLd-0ChJx!PqZsg^#v;k_?<42^A8RJ{^kFTF?Gsh>F*=}TTkzhNK&!2rs z8DCWV8m%O$0_CKX-Pz|g@yV|SzU=~%xVGcu`${@3&{I)Q_Ea$k1=C)|16)clI+JlX z##9qiQA}K;%5nuMhC((@cXMz(5vXOBZ-6Ivxt%=jp@ElGg|CQKS%B+vhu7Pd32^IP z;J&z19yjn@e8SDdi;EqP4#yaha1{^N-X>uid}`&|t5i!XI4(L667Ru>B^=yekv_=6 zlxau&$MZax|LLn+nVZ)`c78$lK#DG?<@};fE>Z^XJujG&U+Y8EllU`x0}a7Yf8txt z9D$KAZWSeDOyFgkeXOs{9IFZW=Bh`a%XSCH}@5MWY z%knzFbT{*HA-!M)NgwGA!$AZ}7%vR~ylu$QA2YE8 zwht9teaQk4caSHK|F|5if2>|QE6NC$HM$1hXb8h__1@8|Gjh;-(JgMdkqL}@cA0s+ z`A+Z?aT+rBMd0)=zh6@g)DZvFyXSTdH>7ll=>6t=PcEwcR&!;ueLQW>E$^4pf+?nf z@`fxD+$}4*5ST5Cy-o{xFMte4p9c%J6m>}PtfK^eZMdh$sBzZ^0B%)p%)aM^xe~72 zf};kIYm~iX%3J|HT^Ew^U8RC?dP!D&H64)59DMitu{3lYzJEBnR0j02`1M4ywc*IF z7j?V`I3e&|qhG}tWjM$*vLDN+!uKa_j`6!FvBcR&_63{$(&O4K+=8ouV4I#d|FoY3 zzkIiwoK)b%gNg@fwD)L3;XJZ>m|_S*>nR_!H>hwU`N}@E&kC@zJpOT>TLT!uBxRaf z4;0QUotR0H2J`6J$FF{|V4WWsTXg0%A%5zCpGO}XJUlrmINYfYqLC(BPF!P#!X%Hp zIW=ae{!o_1Evx`{zP=v&C1U`tQt1jjO}yCSgWtNOoeunYp5q&H<{P=g;41Z_DJoo^ z-_0B$uK}IxN1LZa<)G;l``7^q63`A%2JYbDgIAwKZCT+pS$xNb%8ynOa6rm9h$_?& zHkNn!Ot;biRhq=UVvNAEAL)uPoAeuBgK4U8A2K?v;bq(+nIx zH~DzbykWj(s1DS7RT+-dD#C?_u^z_HRN>K<>6hGJ6v1+|?DHtP#&Mybu@8Ww8xSIbIJ-h9|RwvhzP)eoFo0_5P~ zQPv(RHwvJQzTL`wOa@}nH&tFHX$aaTt2y^h15VnpoUbpC0WHsf_~K2z&YAZ|jdrR5 zdwD5~1-A@TN%q(6oKu4caRy!693?naF2~kq0A7FZLLJ{RCB0bGv+Nxr|OfLi}mw!v#@V3QkEjNfb*+tBx)Z(NcC zzPRRfzb=$^!3#NEJJ$g<-_+{BmQf0^FFjy7^gG8eUXq zS=VJSz&9zj=X1%@U{Ldn+aQ@A6iys`JAOzQEZ!Cd7SGYcjN(tVOUEVPaZB~|`(v_T zc_7cof<+0=%6@;SC@2Hzu6w((gf`F5Bgu!nKEklZd^}I1L<-_Fxu2CXOF_c%(_2d< zMPS5QJmZ))55%UIajJghhT9`fomV!`Kgrm~MJ=5K7Gm1UAs-YC;aGvfmtRei5Ll*M+QK9Ovf20JWaLR8uH1VrH(L_&3@{*jpQ5>EEO5y;Bhun4 zFErj}{zYpo2`{(~?f*!{2j`D`{dmoq1J3VN>Sy003@0gu+fNOVfROqWovc8DFm3NP zM=M_7`@LXCq7Va(K~L(PYQm5{xc*j_fg4=oOz3?XmWbTCmj?{gXyLvX1$V?|`ydhG z!t|UFoU7b1%8?=t&2$B?Jy*HGPW_QNLkS!7(w&*vP0I!v=!5jU_|^Pcz=Jch4 z8;0mD%w;W~Ns;m27fcoP)JIeuxL*!iBXfABjsocl2Ms*Iin66F+c?j-2pXAIxR zYjg|{AYAmi){_m^POnIOlB0zi?-vsf*^QCCD@WTeEi4eD?WqFSROukfIeg1h0xg(W z3;)hr9VCWfzPinj3MMos_f*QQ66CC?NiiM{06`H!6Tx4^Vn?qVZ5Aa+{-jwJ6rCn^ z-0O_#F?dZrEnjtTZ$%|B%DdB%HR3ZdTgBtJt7(cj%sVeBdUJ}rqr&3UE89haxjVx7 z?!r9znvJS1y<0!oRek#h|I7iRx@Ij%%%`7VJkKimET)%un$F`DQ}d0mJ)KaTzdS&m znvlx9)UZh2=NZbrG(SUJ*tpQKdzb=;?0s(h@y9s9W|2`>V*HjYT|n)1+LA9_HYkV$APnCJApGUS?z=53EBzDrle8w7OC8ndd>~9r0nj{9m2Gj z#=x~X&tZ(9QfxE!%45V8a&RjL7O4|83Xs?!+wREEsr|`;{Y0p)S1EAdquEP&-3%P~K|BX5%L#7$ zj8SyS{}>DA6tR)C^`^m}i5nJqI!yREJR7vgXk4v$zjfEI2$N>@7nMH{zW>*DLu(8ir~!tsXw2Ds%Tv+{Q#JKk?O@}H3cl{t0%&v)PAZV}{vif>W=|BJ=9-|RZw$&#TEbV5$s8@XXs zeVZgB215`OlPQ~%*a}fpe|n|(w&1O8{h|F%?(mZ>L-X}ZH#qv9M@?>a2RIDsrV!0e zxNNjAVM{9lzY|{hi+Y+^T9|5SGTQ~t8f^JguFr+Lo_lW3qjkah{*Hl~0}c?HJKXf7 z#t8Fse;T+IjPP*j*~fn1j00$bF3$06!OeF^f4m`Y#VI?>9e2+QVUeS`Ia4nXj92f! z^~2W}G;=%}Wex>lqv0K ztY(Gy5xyNQQ{45~6M_iEXA~khkHh6BbywN0biM*6nI&=5C zG;H(feWfk-))I!Z2AIQ zw@AXAM(xQFA9JYxH8FD#8$kW8s}-pnmiR?mU!;Ge0c6r8 zofG1dgsr77LVUWk;LmF5qq<3_&B?MqSt$$Q1NRj~l&z$2uD1HwSKFOHSWoL)b6x7l4+^Rr#(^kR4yILgiSN-5!PQ1L})oY+|CsGDa|41xYl2ymFSNT78VLNEps?eL; zqy;($Z1o>!>R<-1ZkmiffVq>>)t6tH;jOARWlH{bcyD1_#M?a^WFh>m_B*=53JX8A&%A50DQ@xI_%=#-VcnB0CK}IW;D<6}dB+-pG3G>Xj|EeFRMF?qdtL(=Ub#3C zdQ%wqKR7wq6dFS8%``sREDxM`qjjIFsA7t{6_3>MXZ1JqKOsOm|Q3{Icx;s z3TBkkc?c-`#ta_uap31|BFPq2)>x+BXQ=g=2!ywN%sjNQL|%>PTJolvCv#hUCQkiO z#9Xn1bkrW|@O5eVOQE+D`s`0xh@3jSNa4lU~Y-EEG!}_K(vIJ11n7*3i zb-?7p>xXm{jPQ=)?H(uF)uH3pIof(VbNs0F^OFrdX;|dzOiP_Gfd;k9sef9maO4?} zidF>+P}@4Uv30vJJ|cTb=+0+d2uO~SJ11cVS|1IapIb}fl^|M+-*x=hX(d&@&RZSy zG|c2viQ8GZ-Wr-4z#r>*2ODlnTu+lNkR-0h0*~AEvVQ>L#DnW z2A0JyL+bV@f;emAahfU(IKrIMEdCtf(o?Ek)CaZk{m*j9d0rL_xsEZv$yLIP84O8D z{tO^!b)VmR^Jd|`(;z-YOd9T`ZBq{2oWyZyR%T3Bm;rK~HPBjA!HaJDJ6oq{Abv ziS&QA@BlAa;xJzy7dF2+tgSw;59G6b*+L4k&}K6~V_P8%;?NdtPHPGZC)a-24gDtb zKO5kT@e_g({nBfpn-4RnjC&MsC2k(i%G925Lmj+YzTR&gen74rJC#S9O$i@aWgN_x z7YP5H;~iX^Lyn$P-X~7=$>Yv>DqzG47#8)H?JYNm@JF}Y6t7TYwXvDd%PYosV*{V8 z)1d>Gwy-~%UuEGdnJMnh#024`scaR+MvGI2Usd%vielEUqGm~CSs1J|2_M=e1h(d8 zXA@HviH&aRET4UPcI=8S9qFCS9E&I=y5*$G0&u)?G_KF|8|Z=eQA)KY?(_+ zeXI(-rn{3{`xxQUrE=Pn+a++xMYZ7X`*^XWWrmQ~7z3>Q$@h=iY;R^+ze8giESPDR z39;>h0%Rz=6czLcVXDQNFQfWBL{w6l$IfgWsAKQe(?ZU>{$86`wWqwRQ)Xmgz-@uwTg`G5BU*8{4XDf#Tca}<9P$1j!cx! zm$#$FzZ~2?@x53kjBI)bCm2hK$C0$ow8=Dh;yLSCzL80y2@XA{wBvr3NkQZ8QAS^fBt&?9ATlDS~Xg; zPA2WQq*SpI#2b@graKn6@n;6gEZO`iqH{g2*T#SmJ{Ov0y$Em_TdypsYy` zC$@NOewc|~7$e(1lbC`5bMm?zev4akUUIPFbEWk~?|Q!w0rz*& zOWv1;m=M*WRE=e_c*d76!mnnC?g!MK?A-jYe}2}*gGmHC&%5Wx>t z%=#K<#NkXc-M&lDC2^H`&5o&50VsayxFK+%kKlP#Y0k~f0i@G8$|lQi$$}O&`3HkX z$rc7KYQ@2WhbVNO0&$NBeO1BH8tMVD+)K0ph^Lxn?I9S{!_<{!S%5 zH5`~6x+_k+N*^}+Mc3apF@;h2V^B`wBtXbct z&h?H0%Qb(AY<$EIV=w)jWnI{?@{h{k`fV)W#oQ(|@nMZz@Q6Z_|K})K`xlK)9Z{Z~RV{F6HR?d1``iFSnE1 z#zF~mmpuR6d^tj1Q=9iad0zkvg&eQ@EWiTivfsHWhW#YBP_R_p`0$vV%{#K2xrPBh zKC~DIbu5^DqVz|)Og&)_nc3egC}EKElz?a;75wNr_;=$Kf zL{#|wu5PD0g#KegMP@#F9Orm$kHf{=J+jXKvdisyvyLIAV zcVA^r;ym$srx<&CCN1V>*EYC(V6*%qQJiTO6yViI&Bzl*fk`z14~r72$c%&isH}U1 z>}raiu}syHpG-Hpwd4&DE!)gXuLZ8p%O9J*V{NzY&WO zCBBKrUJ$C@ENOY~8Q}N&U7G71z7x9|0d z8HE@P5U!1xZ`Y)j2-iFHI?s0eAm3$3(^dZbjy$~WP9Im#3nIo-`M!z7JRz@WZ%;aVzRZqDzrMMCcW)~^-=>%^yfk_=6} zKghqX{bHf*o+aPfT|(fqG_a8$zwWbEOQ?(*i$>6l6Q%Z_9M|{$B(jQ*jvtO5Ad0?U zdzK+X1K(fjxxG<+PE`8ea|sVWP9rWbm*Ta?|tS$b-M2- zd7)SC_=iIq#K2LS2g`b^L~lekM_PL~A<{NYeC)3zo<~1e=;jYL{c)elUQ1x|P9 zD|pNj`9B_q_xp_z`)a2u=q|q_S5*q8U0fI-1dI+}f3ztoa&5WjacpxGB=KZ7shxeD zP`nprFk$|UT&{#|dApa$&EDHN1Wxu4EMZiOE7UWDcFW#p)V1~G;!BwnE4P-(F?3$) z5lpQ_{1{KuRn|G;KGss_d)Y%?YSk*sGFl+QXUO~4d_ECdJktd2C1=V0^nR7R7wO>& z&JtC5{+{p`QCaRhR6)MPG-g*UH%ESalOAWcERcQfIqdu_wnz}aO8YxbQNh8|<2vIe zzsMrW)rW^BTL_h?yTS^6-w3qUZ4?^*l?dLo<$$aDIGIoWP3K#~yX5AS#BAlnIij+; zsK-@qmJB1g7s{g!5rZ`4(zU!xL{m(C2YbypAu!T#a<^pM-8iRlelB+$h8Q-Fww%LqZ82l|H^#t37=-BelO1#z$BrnV>f9g%wVihr+57jY5w znY%MELhopWjn&!!AwhjuIX1S7sC@C6M>_XAq56ye>8SoN5$|#nUtF0a3;TK`zu3A? zl=1HovkjUc4zqkN*!Q}Q{M`RXV*WTI&{&C77Y4#Hu-`BuoYPe=Gqm#z}LN$u$r6TA>|@71Tv>*GX0 zeCwk0^cRA?HR&+v90h#GM=tA&(843xF87=KGlclqRLR9^M!0?RHd$Ad19C4*Jj-OE z2leT+J?S5NiCLHS2}OB6(0+f2V@!(;D7!LRlW1As*e6-$t3zCy&cLMahlMyH%W=iH zwuu3n6MYRIyx|7Z8Raf*xjy3B%{@tbOe7$4#^_|%4HAIPlM91x!khE_<6fqF1t7V0 zig*5k8q7R;cPnRI5eWEnCHsOBq$Nbw9_Ua3(T*`g{T3}~AA8R>?4||5PB*G9a%lkQ zwHU5#j@wIZ#JkN}H6cvj`p^=u4h*S;)$41j|A*0b{a}Cm-{Kp?U&hm@i0Hp9)F=LJ zq5eDmk36UUQv)@L#6L34;LEygstqlH)z#-R729#7P)c-WJ|7BGF zYvuT_gZ}k(i1M%apZ50uME_+D{~OKr5A^Kj)<3P{|B3y}X#L+~zipPiFXI2TUH>Qc zFVpg0e&oMa)_;`SN%eQF|1ISHC-kouF8_uy{{!mE_pd7de}I39#s599P2pel{r>>} zO7y=H`LC7XA0^tGY*zN)Y5$+~{42TsdlW127wTWB=3-Av_t#%CY<}7|Ph0!uZxj^& E2NTeo6#xJL literal 0 HcmV?d00001 diff --git a/src/sp_validation/tests/test_b_modes.py b/src/sp_validation/tests/test_b_modes.py index 67c64376..20a9aa67 100644 --- a/src/sp_validation/tests/test_b_modes.py +++ b/src/sp_validation/tests/test_b_modes.py @@ -1,8 +1,9 @@ """VALUE-DRIFT CHARACTERIZATION TESTS FOR THE B-MODE ESTIMATORS. This module pins the numeric behavior of the pure E/B-mode helpers in -``sp_validation.b_modes`` against fixed, deterministic, in-memory inputs -(seeded RNG and hand-built arrays — no cluster data, no catalogue files). +``sp_validation.b_modes`` against fixed, deterministic inputs (seeded RNG, +hand-built arrays and one committed ξ± fixture — no cluster data, no catalogue +files). Every pinned literal was produced by an actual run of the estimator inside the container; a future refactor that changes the numbers must fail. @@ -293,6 +294,78 @@ def test_calculate_eb_statistics_has_teeth(): assert loud_pte < 0.05 # louder B-modes are clearly rejected +# --------------------------------------------------------------------------- +# 5. pure_eb_from_xi on committed ξ± (the transform pin) +# --------------------------------------------------------------------------- + +# pure_eb_from_xi(**fixture); regenerated only when the transform is meant to move. +_PURE_EB_PINS = { + "xip_E": [ + -2.9831529669542025e-06, + -1.5008524620265777e-05, + 3.221623968725757e-07, + 1.1797672310858565e-05, + 5.715510692557323e-06, + 8.825804523824443e-07, + ], + "xim_E": [ + -4.737558091773235e-05, + -0.00010853189443993388, + -9.094825175032069e-05, + -5.826599101284694e-05, + -4.646405415748759e-05, + -1.9978028925333273e-05, + ], + "xip_B": [ + 1.7069121242262332e-05, + 3.059889782373755e-05, + -4.8805399253844115e-06, + -6.999262696335271e-06, + -1.2672006989728095e-05, + -1.214149138979614e-06, + ], + "xim_B": [ + -0.00011478091634539627, + -5.445112002141066e-05, + -3.100806652947907e-05, + -1.0940424256759085e-05, + -5.755185146643215e-06, + -1.628217762504557e-06, + ], + "xip_amb": [ + 0.00014017621792612224, + 0.0001378482153667787, + 0.0001339573019551001, + 0.00012745126271361765, + 0.00011662385105911986, + 9.851844704443032e-05, + ], + "xim_amb": [ + -4.389203999135455e-05, + 5.279277664928643e-05, + 5.800339397836051e-05, + 4.4242350610114584e-05, + 2.9902912946755567e-05, + 1.912262132836568e-05, + ], +} + + +def test_pure_eb_from_xi_reproduces_pins_on_committed_xi(pure_eb_xi): + """The pure-E/B transform of the committed ξ± reproduces its pins. + + With ξ± frozen, these pins move only when the transform does. rtol=1e-6 is + far above the 1e-12 reduction-order noise across thread counts. + """ + modes = b_modes.pure_eb_from_xi(**pure_eb_xi) + for key in b_modes._EB_KEYS: + npt.assert_allclose(modes[key], _PURE_EB_PINS[key], rtol=1e-6, err_msg=key) + + # Teeth: widening the integration interval by 1% leaves the pins. + moved = b_modes.pure_eb_from_xi(**{**pure_eb_xi, "tmax": 1.01 * pure_eb_xi["tmax"]}) + assert not np.allclose(moved["xip_E"], _PURE_EB_PINS["xip_E"], rtol=1e-6, atol=0) + + # --------------------------------------------------------------------------- # 6. Grid edges and the COSEBIs covariance seam # --------------------------------------------------------------------------- diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py deleted file mode 100644 index 890271a4..00000000 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Back-pressure guard #2: the paper Snakemake workflows dry-run. - -The reorg is allowed to change the rule graph; these guards only assert that -Snakemake can still parse each composed workflow and construct a dry run. One -guard covers papers/bmodes (config space, no cosmo_val block); a second covers -papers/cosmo_val, whose config DOES carry a cosmo_val block — so it is the only -one that includes cosmo_val.smk and hence the born-as-SACC + assemble rules. -""" - -import os -import subprocess -import sys -from pathlib import Path - -import pytest - -# The workflow composes a catalog configfile and terminal inputs that live at -# candide-absolute paths (Snakefile line 5, workflow/common.py), so the dry -# run can only be constructed on the cluster. Same pattern as test_cosmo_val. -requires_candide_data = pytest.mark.skipif( - not Path("/n17data/cdaley/unions").exists(), - reason="candide-local workflow config/data (/n17data) absent — off-cluster", -) - - -def _repo_root() -> Path: - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - raise RuntimeError("could not locate repo root (no pyproject.toml above test)") - - -def _dry_run(workflow_dir, targets, *extra_snakemake_args): - """Construct a dry run of the paper workflow at ``workflow_dir``. - - PYTHONUNBUFFERED satisfies the Snakefile's ``envvars:`` declaration. A dry - run never dispatches jobs, so any inherited SNAKEMAKE_PROFILE is dropped - rather than requiring its executor plugin. snakemake is invoked through - sys.executable, since a bare python3.12 may resolve off PATH to an - interpreter without it. - """ - env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} - env.pop("SNAKEMAKE_PROFILE", None) - return subprocess.run( - [ - sys.executable, - "-m", - "snakemake", - *targets, - "--dry-run", - "--cores", - "1", - "--configfile", - "config/config.yaml", - *extra_snakemake_args, - ], - cwd=workflow_dir, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=120, - check=False, - ) - - -@requires_candide_data -def test_bmodes_workflow_dry_runs(): - """The paper B-mode workflow must still parse and dry-run cleanly.""" - result = _dry_run(_repo_root() / "papers/bmodes", ["paper"]) - assert result.returncode == 0, result.stdout - - -@requires_candide_data -def test_cosmo_val_workflow_assemble_dry_runs(): - """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the - born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov.""" - version = "SP_v1.4.6.3_leak_corr" - result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) - assert result.returncode == 0, result.stdout - # assemble_sacc must pull the tagged pseudo-Cl part + its NaMaster - # covariance (not the untagged cv_pseudo_cl diagnostic), plus every part. - out = result.stdout - assert "rule assemble_sacc:" in out, out - assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out - assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out - for part in ("_xi_minsep=", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): - assert part in out, f"missing {part} part in assemble DAG:\n{out}" diff --git a/src/sp_validation/tests/test_config_paths_exist.py b/src/sp_validation/tests/test_config_paths_exist.py index 2db03bc1..38a269ec 100644 --- a/src/sp_validation/tests/test_config_paths_exist.py +++ b/src/sp_validation/tests/test_config_paths_exist.py @@ -182,12 +182,31 @@ def _config_files() -> list[Path]: ] +def _located_elsewhere(source: Path, key: str, value: str) -> bool: + """Whether a path-shaped value names nothing the config itself locates. + + ``paths.output`` in the catalogue config is where cosmo_val writes, created + by the run. A calibration ``params.input_path`` with no directory is opened + relative to its consumer's run directory (the image-sim run, or the + catalogue directory ``scripts/masking.py`` prefixes). + """ + if source.name == "cat_config.yaml": + return key == "paths.output" + return ( + source.parent.name == "calibration" + and key == "params.input_path" + and "/" not in value + ) + + def _candidate_paths() -> list[tuple[Path, str, Path]]: root = _repo_root() candidates = [] for config_path in _config_files(): iterator = _iter_ini_paths if config_path.suffix == ".ini" else _iter_yaml_paths for source, key, value, base_dir in iterator(config_path): + if _located_elsewhere(source, key, value): + continue expanded = Path(value).expanduser() if expanded.is_absolute(): resolved = expanded diff --git a/src/sp_validation/tests/test_container_smoke.py b/src/sp_validation/tests/test_container_smoke.py deleted file mode 100644 index b9b07058..00000000 --- a/src/sp_validation/tests/test_container_smoke.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Smoke test of the profile-driven containerized-SLURM path. - -Submits one real (tiny, 5-minute) SLURM job through the committed candide -profile. The executor, the apptainer deployment method and the bind mounts come -from that profile; the image is the module-level ``container:`` in the test -Snakefile, exactly as real workflows declare it. That contract is what's under -test, so this can only run on candide -- marked ``slow``, skipped elsewhere. - -The job writes a YAML report (see data/container_smoke/container_smoke.py); the -assertions below check what it reports. -""" - -import os -import re -import shutil -import subprocess -import tempfile -from pathlib import Path - -import numpy as np -import pytest -import yaml - -requires_cluster = pytest.mark.skipif( - not Path("/n17data/cdaley/unions").exists() or shutil.which("sbatch") is None, - reason="needs candide: /n17data and a SLURM submit host", -) - - -def _repo_root() -> Path: - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - raise RuntimeError("could not locate repo root (no pyproject.toml above test)") - - -def _reference_eigenvalues() -> np.ndarray: - """The same deterministic computation the job runs inside the container.""" - rng = np.random.default_rng(seed=42) - a = rng.standard_normal((8, 8)) - return np.linalg.eigh(a + a.T)[0] - - -def test_smoke_snakefile_names_the_workflow_image(): - """The test Snakefile's literal image must track the package's CONTAINER_URI.""" - repo_root = _repo_root() - uri = re.search( - r'^CONTAINER_URI = "(.+)"$', - (repo_root / "src/sp_validation/container.py").read_text(), - re.MULTILINE, - ).group(1) - snakefile = ( - repo_root / "src/sp_validation/tests/data/container_smoke/Snakefile" - ).read_text() - assert f'"{uri}"' in snakefile, uri - - -@pytest.mark.slow -@requires_cluster -def test_container_smoke(): - repo_root = _repo_root() - workflow_dir = repo_root / "src/sp_validation/tests/data/container_smoke" - - # Not pytest's tmp_path: that lives in the login node's /tmp, which the - # compute node cannot see, so the job's output would "go missing". The - # workdir must be on a shared filesystem. - tmp_path = Path(tempfile.mkdtemp(prefix="container_smoke_", dir=Path.home())) - - env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} - result = subprocess.run( - [ - "snakemake", - "--profile", - str(repo_root / "workflow/profiles/candide"), - "-s", - str(workflow_dir / "Snakefile"), - "--directory", - str(tmp_path), - "--jobs", - "1", - "container_smoke", - ], - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=600, - check=False, - ) - assert result.returncode == 0, result.stdout - - report = yaml.safe_load((tmp_path / "results/container_smoke.yaml").read_text()) - - # The job ran inside the image, not on the bare host. Everything below would - # pass on the host too, so this is the assertion that makes them mean - # something: apptainer sets APPTAINER_CONTAINER in every process it starts. - assert report["container"]["apptainer_container"] != "unset", report["container"] - - # The install must resolve to an editable src/ checkout, not a site-packages - # copy. Note it need not be *this* checkout: the container's editable install - # points at the shared /n17data working tree, while the Snakefile under test - # is read from wherever the test runs. - module_file = Path(report["sp_validation"]["file"]) - assert module_file.parts[-3:] == ("src", "sp_validation", "__init__.py"), ( - module_file - ) - assert "site-packages" not in module_file.parts, module_file - - # The numeric stack agrees with the same computation run here. - np.testing.assert_allclose( - report["numeric"]["eigenvalues"], - _reference_eigenvalues(), - rtol=1e-10, - atol=1e-12, - ) - - # numeric.omp_num_threads is recorded but deliberately NOT asserted: the - # profile leaves OMP_NUM_THREADS unset by design, and rules that need it - # pinned set it themselves, so "unset" here is correct rather than a gap. - - # git worked inside the container, so /home is bound and usable. - assert re.fullmatch(r"[0-9a-f]{40}", report["provenance"]["commit"]), report[ - "provenance" - ] - - shutil.rmtree(tmp_path) # keep only on failure, for post-mortem diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index f25ed522..50f046f0 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -9,9 +9,7 @@ """ import os -from collections import defaultdict from pathlib import Path -from typing import Dict, Iterator, Tuple import numpy as np import pytest @@ -182,88 +180,6 @@ def test_additive_bias_leak_corrected_columns(self, base_config): assert isinstance(cv.c1[version_leak_corr], float) assert isinstance(cv.c2[version_leak_corr], float) - @staticmethod - def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: - """Yield (name, entry) pairs for catalog-like entries in the config.""" - for name, entry in config.items(): - if not isinstance(entry, dict): - continue - if "subdir" not in entry: - continue - yield name, entry - - @staticmethod - def _resolve(base: Path, candidate: str) -> Path: - """Return an absolute path given a base directory and a candidate string.""" - candidate_path = Path(candidate) - return candidate_path if candidate_path.is_absolute() else base / candidate_path - - @pytest.mark.slow - @requires_catalog_data - def test_catalog_paths_exist(self, base_config): - """Verify that catalog paths for active versions exist on disk. - - This is a lightweight test that checks that all files referenced in the - catalog configuration for UNIONS analysis versions actually exist. It - discovers versions programmatically from cat_config.yaml rather than - using hardcoded lists. - """ - # Get the path to catalog config - repo_root = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - ) - catalog_config_path = os.path.join(repo_root, "cosmo_val", "cat_config.yaml") - - config = yaml.safe_load(Path(catalog_config_path).read_text()) - - # This integrity check needs the real catalogs on disk (cluster only). - # Skip where the data directories aren't mounted — e.g. CI running - # inside the docker image, which has cat_config.yaml but no catalogs. - if not any( - Path(entry["subdir"]).is_dir() - for _, entry in self._iter_catalog_entries(config) - ): - pytest.skip("catalog data directories not present (not on cluster)") - - working = [] - nonfunctional = defaultdict(set) - - for version, entry in self._iter_catalog_entries(config): - # Skip nz entries and versions already tested in heavy tests - if version == "nz": - continue - - base = Path(entry["subdir"]) - version_missing = set() - - # Check shear, star, and psf files - for block_name in ("shear", "star", "psf"): - block = entry.get(block_name) - if not block: - continue - resolved_path = self._resolve(base, block["path"]) - if not resolved_path.is_file(): - version_missing.add(block_name) - - if version_missing: - nonfunctional[version] = version_missing - else: - working.append(version) - - # Print summary - print(f"\n✓ Working versions ({len(working)}):") - for v in sorted(working): - print(f" - {v}") - - if nonfunctional: - print(f"\n✗ Non-functional versions ({len(nonfunctional)}):") - for v in sorted(nonfunctional.keys()): - print(f" - {v}: missing {nonfunctional[v]}") - - assert not nonfunctional, ( - f"Catalog configuration references missing files: {dict(nonfunctional)}" - ) - def test_seed_variant_updates_shear_path(self, tmp_path): """Seeded versions should materialize a seed-specific shear path.""" params, base_version = self._make_seed_config( @@ -346,11 +262,11 @@ def test_v1_4_6_glass_mock_default_seed(self, base_config): # These run the real compute seams end-to-end on a small, deterministic # toy catalog written to disk, asserting that sp_validation wires the # catalog/config/estimator together correctly and that the chain produces - # output of the right shape with finite values. They do NOT re-test the - # underlying numerical libraries (treecorr, cosmo_numba): no specific - # numerical values are asserted. These are the back-pressure that catches - # config-path / wiring breakage during restructuring. They can be tightened - # to allclose-against-a-committed-reference later for value-drift coverage. + # output of the right shape with finite values; a test that also compares + # values says against what in its docstring. They do NOT re-test the + # underlying numerical libraries (treecorr, cosmo_numba). These are the + # back-pressure that catches config-path / wiring breakage during + # restructuring. # # Environment-independent: the catalog is synthesized in a tmp dir, so no # cluster data is needed. They do require the scientific stack (treecorr, @@ -438,6 +354,7 @@ def _write_synthetic_catalogs( shear_cfg = { "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", "e2_col": "e2", @@ -522,6 +439,102 @@ def test_calculate_2pcf_runs_on_synthetic_catalog(self, tmp_path): # The additive-bias subtraction in the pipeline must have run. assert version in cv.c1 and version in cv.c2 + @pytest.mark.parametrize("npatch", [1, 4]) + def test_xi_part_carries_the_covariance_the_measurement_estimated( + self, tmp_path, npatch + ): + """run_2pcf's ξ± part carries TreeCorr's covariance, whoever calls it. + + The jackknife covariance with patches, the shot-noise diagonal without, + so every part has variances whether the rule or the CLI measured it. + """ + import importlib.util + + import sacc + + from sp_validation import sacc_io + + script = Path(__file__).resolve().parents[3] / "workflow/scripts/run_2pcf.py" + spec = importlib.util.spec_from_file_location("run_2pcf_part", script) + run_2pcf = importlib.util.module_from_spec(spec) + spec.loader.exec_module(run_2pcf) + + params, version = self._write_synthetic_catalogs(tmp_path) + part = tmp_path / "part.sacc" + gg = run_2pcf.run_2pcf( + ver=version, + min_sep=5.0, + max_sep=100.0, + nbins=6, + npatch=npatch, + cat_config=params["catalog_config"], + output_dir=params["output_dir"], + sacc_out=str(part), + ) + + cov = sacc_io.load(str(part), allow_unblinded=True).covariance + if npatch > 1: + assert isinstance(cov, sacc.covariance.FullCovariance) + np.testing.assert_array_equal(cov.dense, gg.cov) + else: + assert isinstance(cov, sacc.covariance.DiagonalCovariance) + np.testing.assert_array_equal( + cov.diag, np.concatenate([gg.varxip, gg.varxim]) + ) + + def test_a_patched_xi_dump_reads_back(self, tmp_path): + """calculate_2pcf reads back the text dump a patched measurement wrote. + + The ξ± figure rules re-enter calculate_2pcf on the reporting grid, which + has patches, and are handed the dump rule xi wrote (to its precision). + """ + params, version = self._write_synthetic_catalogs(tmp_path) + binning = dict(npatch=4, min_sep=5.0, max_sep=100.0, nbins=6) + measured = CosmologyValidation(versions=[version], **params).calculate_2pcf( + version, **binning + ) + read = CosmologyValidation(versions=[version], **params).calculate_2pcf( + version, **binning + ) + for column in ("meanr", "npairs", "xip", "xim", "varxip", "varxim"): + np.testing.assert_allclose( + getattr(read, column), getattr(measured, column), rtol=1e-4 + ) + + def test_calculate_2pcf_does_not_depend_on_thread_count(self, tmp_path): + """calculate_2pcf's ξ± is the same on 4 and on 48 TreeCorr threads. + + Production binning (default bin_slop/angle_slop), both runs on the + jackknife patches the first one writes, each from a fresh Catalog; they + must agree to far below the jackknife σ. + """ + import treecorr + + params, version = self._write_synthetic_catalogs( + tmp_path, n_gal=4000, coherent_shear=True + ) + cv = CosmologyValidation( + versions=[version], + npatch=8, + theta_min=15.0, + theta_max=70.0, + nbins=6, + **params, + ) + + xi = {} + for n_threads in (4, 48): + # calculate_2pcf reads back an existing text dump instead of measuring. + for dump in Path(params["output_dir"]).glob(f"{version}_xi_*.txt"): + dump.unlink() + gg = cv.calculate_2pcf(version, num_threads=n_threads) + assert treecorr.get_omp_threads() == n_threads # the count took effect + xi[n_threads] = np.concatenate([gg.xip, gg.xim]) + sigma = np.sqrt(np.concatenate([gg.varxip, gg.varxim])) + + shift = np.max(np.abs(xi[48] - xi[4]) / sigma) + assert shift < 1e-6, f"ξ± moves by {shift:.3g}σ between 4 and 48 threads" + def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( self, tmp_path ): @@ -557,49 +570,27 @@ def test_calculate_scale_dependent_leakage_runs_on_synthetic_catalog( assert np.all(np.isfinite(res.alpha_leak)) assert hasattr(res, "C_sys_p") and hasattr(res, "C_sys_m") - def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): - """calculate_pure_eb wires xi+/- into cosmo_numba's Schneider E/B split. - - Integration test of the headline B-mode seam: the pure E/B/amb - decomposition runs end-to-end via cosmo_numba and returns vectors of the - configured length, all bins finite, with a jackknife covariance of the - right shape -- AND the deterministic mode vectors match pinned reference - values, so a refactor that silently changes the numerical B-modes fails - rather than staying green on finiteness alone. - - Two layers of teeth: - - 1. Finiteness on EVERY reporting bin (not just the interior). The - Schneider (2022) pure E/B estimator evaluates singular kernel - integrals (Eq. 42-43, 55-56) at each reporting theta, integrating the - fine ``gg_int`` xi+/- over [tmin, tmax]. At the extreme reporting bins - the evaluation point sits at the integration boundary, where the - integrand is near-singular; a *coarse* integration grid fails to - resolve it and the mode goes NaN. The real bmodes workflow - (papers/bmodes/config.yaml) avoids this with a broad-and-fine grid -- - reporting [1, 250] arcmin, integration [0.5, 300] with nbins_int=1000 - -- so the integration range brackets the reporting range AND the grid - is fine enough that the boundary integrals converge. This test mirrors - that: reporting [15, 70] arcmin, integration [1, 300] arcmin (brackets - on both ends) with nbins_int=600. Confirmed directly that nbins_int~80 - over this range NaNs the last xip_E bin and the first xim_E bin, so - coarsening the integration grid back toward ~80 reintroduces edge NaNs - and fails -- this is the finiteness teeth. - - 2. Value-drift pins on the four deterministic mode vectors (xip/xim, - E/B). These come from a seeded synthetic catalog -> full-sample - treecorr xi+/- (no RNG) -> Schneider linear transform, so they are - reproducible. Verified bitwise-stable across two separate container - processes to a worst-case relative drift of ~1.4e-11 (pure float64 - reduction-order noise; absolute drift ~1.5e-17). The pins use - rtol=1e-6 / atol=1e-12 -- ~5 orders of magnitude above that float-noise - floor (no flakiness margin consumed) yet tight enough that a sub- - percent change in any mode bites. The jackknife COVARIANCE depends on - treecorr's kmeans patch assignment and is NOT pinned by value -- only - its shape is asserted. + def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path, pure_eb_xi): + """calculate_pure_eb carries ξ± through cosmo_numba's pure-E/B split. + + The ξ± it measures equal the committed ``pure_eb_xi``, its modes are + ``pure_eb_from_xi`` of those ξ± and edges, and every reporting bin is + finite. ``test_b_modes`` pins the transform itself on the same ξ±, so a + failure names the step that moved: measurement, wiring or transform. + + Finiteness: the Schneider (2022) integrals are near-singular where a + reporting bin meets the integration boundary, so the integration grid + [1, 300]′ brackets the reporting grid [15, 70]′ on both ends and is fine + (600 bins); about 80 integration bins NaN the edge bins. + + ξ±: exact binning (bin_slop = angle_slop = 0) makes ξ± a plain pair sum, + independent of the tree and so of the jackknife patches, whose k-means + centres this test does not fix. """ pytest.importorskip("treecorr") pytest.importorskip("cosmo_numba") + from sp_validation import b_modes + # Coherent shear -> smooth xi+/-, so the pure-E/B integral is well-posed. params, version = self._write_synthetic_catalogs( tmp_path, n_gal=4000, coherent_shear=True @@ -615,13 +606,8 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): nbins=nbins, **params, ) + cv.treecorr_config.update(bin_slop=0, angle_slop=0) - # Integration range strictly brackets the reporting range [15, 70] on - # both ends (1 << 15, 300 >> 70) AND uses a fine grid (nbins_int=600), so - # the near-singular boundary-bin Schneider integrals converge. This - # mirrors the bmodes workflow's broad-and-fine integration grid; every - # reporting bin is well-defined (no edge NaNs). nbins_int~80 here would - # NaN the edge bins -- confirmed -- which is the finiteness teeth. results = cv.calculate_pure_eb( version, npatch=npatch, @@ -630,71 +616,30 @@ def test_calculate_pure_eb_runs_on_synthetic_catalog(self, tmp_path): nbins_int=600, ) - # Reference mode vectors from the seeded synthetic catalog + Schneider - # transform. Deterministic (full-sample treecorr, no RNG); regenerate by - # running calculate_pure_eb with the setup above and printing repr() of - # results[key]. Tolerances justified in the docstring. - expected = { - "xip_E": np.array( - [ - 1.6688018692521218e-06, - -1.8392317186434428e-05, - 1.4170916007248522e-06, - 8.1454486560987474e-06, - 6.2050467269160570e-06, - 2.6649478149110497e-06, - ] - ), - "xim_E": np.array( - [ - -4.4552381788304276e-05, - -1.1082898248960663e-04, - -9.2495668600755951e-05, - -5.8456322151105526e-05, - -4.4270469941501174e-05, - -2.4236697154723798e-05, - ] - ), - "xip_B": np.array( - [ - 1.8508958599700601e-05, - 3.8264056862769537e-05, - -1.0482698132038303e-05, - -7.3081832089716533e-06, - -9.1621105374021936e-06, - -6.4075815485457576e-06, - ] - ), - "xim_B": np.array( - [ - -1.1129938750754923e-04, - -4.7967477760986883e-05, - -3.4334760596175194e-05, - -1.4776328993077835e-05, - -4.0078671892721522e-06, - -8.3202301900417799e-07, - ] - ), + measured = { + "theta_report": results["theta"], + "xip_report": results["xip"], + "xim_report": results["xim"], + "theta_int": results["theta_int"], + "xip_int": results["xip_int"], + "xim_int": results["xim_int"], + "tmin": results["left_edges"][0], + "tmax": results["right_edges"][-1], } + # Regenerate the fixture with np.savez(conftest.PURE_EB_XI, **measured). + for key, value in measured.items(): + np.testing.assert_allclose( + value, pure_eb_xi[key], rtol=1e-10, atol=0, err_msg=key + ) - for key in ("xip_E", "xim_E", "xip_B", "xim_B"): + modes = b_modes.pure_eb_from_xi(**measured) + for key in b_modes._EB_KEYS: vec = np.asarray(results[key]) assert vec.shape == (nbins,) - # All reporting bins are well-defined under the widened integration - # range (no edge NaNs) -- the finiteness teeth. assert np.all(np.isfinite(vec)), f"{key} not finite" - # Value-drift pins -- the deterministic-mode teeth. - np.testing.assert_allclose( - vec, - expected[key], - rtol=1e-6, - atol=1e-12, - err_msg=f"{key} drifted from pinned reference", - ) + np.testing.assert_allclose(vec, modes[key], rtol=1e-10, err_msg=key) # Jackknife covariance over the 6 stats (xip/xim x E/B/amb) x nbins. - # Patch (kmeans) assignment isn't guaranteed deterministic, so only the - # shape is pinned, not the values. cov = np.asarray(results["cov"]) assert cov.shape == (6 * nbins, 6 * nbins) assert results["n_eff"] == npatch diff --git a/src/sp_validation/tests/test_cv_init_params.py b/src/sp_validation/tests/test_cv_init_params.py index f769c179..240f9213 100644 --- a/src/sp_validation/tests/test_cv_init_params.py +++ b/src/sp_validation/tests/test_cv_init_params.py @@ -17,7 +17,6 @@ REPO = Path(__file__).resolve().parents[3] EXEMPT = { - "output_dir": "rules set the output tree via the run directory / COSMO_VAL", "blind": "None keeps the n(z) blind declared in the catalogue config", } diff --git a/src/sp_validation/tests/test_xi_grids.py b/src/sp_validation/tests/test_xi_grids.py index c35a2e3f..60361d10 100644 --- a/src/sp_validation/tests/test_xi_grids.py +++ b/src/sp_validation/tests/test_xi_grids.py @@ -34,12 +34,7 @@ def _load_common(): "nbins": 20, "npatch": 100, "integration": {"min_sep": 0.08, "max_sep": 300, "nbins": 1000}, - "cosebis": { - "min_sep_int": 0.9, - "max_sep_int": 300, - "nbins_int": 1000, - "npatch": 100, - }, + "cosebis": {"nmodes": 20, "scale_cuts": [[12, 83]]}, } } FIDUCIAL = { @@ -61,15 +56,11 @@ def test_tag_is_built_from_canonical_values(): for a path the producer never writes. """ grids = common.xi_grids(CONFIG, FIDUCIAL) - assert common.grid_binning(grids["integration"]).endswith( - "minsep=0.08_maxsep=300.0_nbins=1000_npatch=1" - ) + # Counts stay integers, so no "nbins=1000.0" creeps into a name. assert ( - common.grid_binning(grids["cosebis"]) - == "minsep=0.9_maxsep=300.0_nbins=1000_npatch=100" + common.grid_binning(grids["integration"]) + == "minsep=0.08_maxsep=300.0_nbins=1000_npatch=1" ) - # Counts stay integers, so no "nbins=1000.0" creeps into a name. - assert "nbins=1000_" in common.grid_binning(grids["cosebis"]) def test_grid_lookup_round_trips_through_the_tag(): @@ -86,14 +77,6 @@ def test_grid_lookup_round_trips_through_the_tag(): assert common.grid_of(grids, {k: str(v) for k, v in binning.items()}) == name -def test_covariance_mode_follows_the_patches(): - """Patched grids get a jackknife block, unpatched ones none.""" - grids = common.xi_grids(CONFIG, FIDUCIAL) - assert grids["reporting"]["cov"] == "jackknife" - assert grids["cosebis"]["cov"] == "jackknife" - assert grids["integration"]["cov"] == "none" - - def test_unnamed_binning_is_a_reporting_measurement(): """The paper's convergence-check binning belongs to no named grid.""" grids = common.xi_grids(CONFIG, FIDUCIAL) @@ -106,4 +89,3 @@ def test_workflow_without_cosmo_val_falls_back_to_fiducial(): grids = common.xi_grids({}, FIDUCIAL) assert grids["reporting"]["npatch"] == 1 assert grids["integration"]["min_sep"] == 0.5 - assert "cosebis" not in grids diff --git a/uv.lock b/uv.lock index 1460cbc9..4acc2e41 100644 --- a/uv.lock +++ b/uv.lock @@ -94,15 +94,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, ] -[[package]] -name = "argparse-dataclass" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/ff/a2e4e328075ddef2ac3c9431eb12247e4ba707a70324894f1e6b4f43c286/argparse_dataclass-2.0.0.tar.gz", hash = "sha256:09ab641c914a2f12882337b9c3e5086196dbf2ee6bf0ef67895c74002cc9297f", size = 6395, upload-time = "2023-06-11T20:32:54.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/66/e6c0a808950ba5a4042e2fcedd577fc7401536c7db063de4d7c36be06f84/argparse_dataclass-2.0.0-py3-none-any.whl", hash = "sha256:3ffc8852a88d9d98d1364b4441a712491320afb91fb56049afd8a51d74bb52d2", size = 8762, upload-time = "2023-06-11T20:32:52.724Z" }, -] - [[package]] name = "array-api-compat" version = "1.15.0" @@ -480,33 +471,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] -[[package]] -name = "conda-inject" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/a8/8dc86113c65c949cc72d651461d6e4c544b3302a85ed14a5298829e6a419/conda_inject-1.3.2.tar.gz", hash = "sha256:0b8cde8c47998c118d8ff285a04977a3abcf734caf579c520fca469df1cd0aac", size = 3635, upload-time = "2024-05-27T12:20:58.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4c/fc30b69fb4062aee57e3ab7ff493647c4220144908f0839c619f912045bf/conda_inject-1.3.2-py3-none-any.whl", hash = "sha256:6e641b408980c2814e3e527008c30749117909a21ff47392f07ef807da93a564", size = 4133, upload-time = "2024-05-27T12:20:57.332Z" }, -] - -[[package]] -name = "configargparse" -version = "1.7.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/30328302903c55218ffc5199646d0e9d28348ff26c02ba77b2ffc58d294a/configargparse-1.7.5.tar.gz", hash = "sha256:e3f9a7bb6be34d66b2e3c4a2f58e3045f8dfae47b0dc039f87bcfaa0f193fb0f", size = 53548, upload-time = "2026-03-11T02:19:38.144Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/19/3ba5e1b0bcc7b91aeab6c258afd70e4907d220fed3972febe38feb40db30/configargparse-1.7.5-py3-none-any.whl", hash = "sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592", size = 27692, upload-time = "2026-03-11T02:19:36.442Z" }, -] - -[[package]] -name = "connection-pool" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/df/c9b4e25dce00f6349fd28aadba7b6c3f7431cc8bd4308a158fbe57b6a22e/connection_pool-0.0.3.tar.gz", hash = "sha256:bf429e7aef65921c69b4ed48f3d48d3eac1383b05d2df91884705842d974d0dc", size = 3795, upload-time = "2020-09-17T02:48:28.824Z" } - [[package]] name = "contourpy" version = "1.3.3" @@ -858,15 +822,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, ] -[[package]] -name = "dpath" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/e1fd64d36e4a5717bd5e6b2ad188f5eaa2e902fde871ea73a79875793fc9/dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e", size = 28266, upload-time = "2024-06-12T22:08:03.686Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/d1/8952806fbf9583004ab479d8f58a9496c3d35f6b6009ddd458bdd9978eaf/dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576", size = 17618, upload-time = "2024-06-12T22:08:01.881Z" }, -] - [[package]] name = "ducc0" version = "0.41.0" @@ -1070,30 +1025,6 @@ dependencies = [ { name = "scipy" }, ] -[[package]] -name = "gitdb" -version = "4.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smmap" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, -] - -[[package]] -name = "gitpython" -version = "3.1.50" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "gitdb" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, -] - [[package]] name = "glass" version = "2025.3" @@ -1138,44 +1069,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, ] -[[package]] -name = "greenlet" -version = "3.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, - { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, - { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, - { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, - { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, - { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, - { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" }, - { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" }, - { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" }, - { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" }, - { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" }, - { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" }, - { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" }, - { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" }, - { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" }, - { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" }, - { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" }, - { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" }, - { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" }, - { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" }, - { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" }, - { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" }, - { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" }, -] - [[package]] name = "gsl" version = "0.0.3" @@ -1354,15 +1247,6 @@ http2 = [ { name = "h2" }, ] -[[package]] -name = "humanfriendly" -version = "10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, -] - [[package]] name = "hyperframe" version = "6.1.0" @@ -1403,22 +1287,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, ] -[[package]] -name = "immutables" -version = "0.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/41/0ccaa6ef9943c0609ec5aa663a3b3e681c1712c1007147b84590cec706a0/immutables-0.21.tar.gz", hash = "sha256:b55ffaf0449790242feb4c56ab799ea7af92801a0a43f9e2f4f8af2ab24dfc4a", size = 89008, upload-time = "2024-10-10T00:55:01.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/52/cb9e2bb7a69338155ffabbd2f993c968c750dd2d5c6c6eaa6ebb7bfcbdfa/immutables-0.21-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48b116aaca4500398058b5a87814857a60c4cb09417fecc12d7da0f5639b73d", size = 104270, upload-time = "2024-10-10T00:54:21.912Z" }, - { url = "https://files.pythonhosted.org/packages/0f/a4/25df835a9b9b372a4a869a8a1ac30a32199f2b3f581ad0e249f7e3d19eed/immutables-0.21-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad7c0c74b285cc0e555ec0e97acbdc6f1862fcd16b99abd612df3243732e741", size = 104864, upload-time = "2024-10-10T00:54:22.956Z" }, - { url = "https://files.pythonhosted.org/packages/4a/51/b548fbc657134d658e179ee8d201ae82d9049aba5c3cb2d858ed2ecb7e3f/immutables-0.21-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e44346e2221a5a676c880ca8e0e6429fa24d1a4ae562573f5c04d7f2e759b030", size = 99733, upload-time = "2024-10-10T00:54:23.99Z" }, - { url = "https://files.pythonhosted.org/packages/47/db/d7b1e0e88faf07fe9a88579a86f58078a9a37fff871f4b3dbcf28cad9a12/immutables-0.21-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b10139b529a460e53fe8be699ebd848c54c8a33ebe67763bcfcc809a475a26f", size = 101698, upload-time = "2024-10-10T00:54:25.734Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d0/a5fb7c164ddb298ec37537e618b70dfa30c7cae9fac01de374c36489cbc9/immutables-0.21-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:583d2a63e444ce1538cc2bda56ae1f4a1a11473dbc0377c82b516bc7eec3b81e", size = 104334, upload-time = "2024-10-10T00:54:31.284Z" }, - { url = "https://files.pythonhosted.org/packages/f3/a5/5fda0ee4a261a85124011ac0750fec678f00e1b2d4a5502b149a3b4d86d9/immutables-0.21-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b274a52da9b106db55eceb93fc1aea858c4e6f4740189e3548e38613eafc2021", size = 104898, upload-time = "2024-10-10T00:54:32.295Z" }, - { url = "https://files.pythonhosted.org/packages/93/fa/d46bfe92f2c66d35916344176ff87fa839aac9c16849652947e722b7a15f/immutables-0.21-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:338bede057250b33716a3e4892e15df0bf5a5ddbf1d67ead996b3e680b49ef9e", size = 99966, upload-time = "2024-10-10T00:54:34.046Z" }, - { url = "https://files.pythonhosted.org/packages/d7/f5/2a19e2e095f7a39d8d77dcc10669734d2d99773ce00c99bdcfeeb7d714e6/immutables-0.21-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8781c89583b68f604cf30f0978b722165824c3075888639fde771bf1a3e12dc0", size = 101773, upload-time = "2024-10-10T00:54:35.851Z" }, -] - [[package]] name = "importlib-metadata" version = "9.0.0" @@ -2853,15 +2721,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, ] -[[package]] -name = "pulp" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/70/69be07a67621ad804d6cf347965eb4e0d7786a97330d99c31d735aaa6c5a/pulp-3.3.2.tar.gz", hash = "sha256:d0904700c207ac11e25e3b1213b70eae1d6fb25faa719d75f3f15054901258c0", size = 16305346, upload-time = "2026-05-25T09:41:26.207Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/6e/d674f1dde91c71e2ac19e5e5cf1ee6d5845e3aefecd9c39ed9c4b0c9a696/pulp-3.3.2-py3-none-any.whl", hash = "sha256:631b166f72086971a9597f7a0233ababa99bb8d50a01cd543f7758be5a9f86c0", size = 16391742, upload-time = "2026-05-25T09:41:22.2Z" }, -] - [[package]] name = "pure-eval" version = "0.2.3" @@ -3646,27 +3505,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/ae/fa6cd331b364ad2bbc31652d025f5747d89cbb75576733dfdf8efe3e4d62/slicerator-1.1.0-py3-none-any.whl", hash = "sha256:167668d48c6d3a5ba0bd3d54b2688e81ee267dc20aef299e547d711e6f3c441a", size = 10274, upload-time = "2022-04-07T18:54:07.029Z" }, ] -[[package]] -name = "smart-open" -version = "7.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/c6/22e7a2acd5d27941e85e0d7ede398da5abe2e4677d2265c924157247c32e/smart_open-7.7.1.tar.gz", hash = "sha256:9414ba5733e28309f29b28a303b0f1054ad23fe0275f1a1b600c80a724f4bd1a", size = 54952, upload-time = "2026-06-26T07:56:35.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/c5/14732c140fa0af0e59345949eaf9016c75ad3f076586f2413e3d3566a8da/smart_open-7.7.1-py3-none-any.whl", hash = "sha256:cb62dc45f519bf39b612564d326d2a17556f3a6056e3c3a86a07215d670d45bc", size = 65061, upload-time = "2026-06-26T07:56:33.685Z" }, -] - -[[package]] -name = "smmap" -version = "5.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, -] - [[package]] name = "smokescreen" version = "1.5.6" @@ -3682,128 +3520,6 @@ dependencies = [ { name = "scipy" }, ] -[[package]] -name = "snakemake" -version = "9.23.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "conda-inject" }, - { name = "configargparse" }, - { name = "connection-pool" }, - { name = "docutils" }, - { name = "dpath" }, - { name = "gitpython" }, - { name = "humanfriendly" }, - { name = "immutables" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "nbformat" }, - { name = "packaging" }, - { name = "platformdirs" }, - { name = "psutil" }, - { name = "pulp" }, - { name = "pyyaml" }, - { name = "referencing" }, - { name = "requests" }, - { name = "smart-open" }, - { name = "snakemake-interface-common" }, - { name = "snakemake-interface-executor-plugins" }, - { name = "snakemake-interface-logger-plugins" }, - { name = "snakemake-interface-report-plugins" }, - { name = "snakemake-interface-scheduler-plugins" }, - { name = "snakemake-interface-storage-plugins" }, - { name = "sqlmodel" }, - { name = "tabulate" }, - { name = "tenacity" }, - { name = "throttler" }, - { name = "wrapt" }, - { name = "yte" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/c2/45aa858e55edbb7dbc243ed60859f4f3d92d63ecdd67a80632aa3659fe6b/snakemake-9.23.1.tar.gz", hash = "sha256:ef8d698bfce66a6669cc29df7e344b0b367fd90956c725e5ae0a79a556e8e93f", size = 6802587, upload-time = "2026-06-18T09:42:57.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/65/2cb1c99117746bea8aff34dc434703f8f23d3876860e2ff3ebeccb754b0f/snakemake-9.23.1-py3-none-any.whl", hash = "sha256:9fdbfff25f1b442cdf7906ca06b465fc24b9bf72e2d51514fb93d83e3f8aeb35", size = 1162133, upload-time = "2026-06-18T09:42:55.541Z" }, -] - -[[package]] -name = "snakemake-interface-common" -version = "1.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass" }, - { name = "configargparse" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/c3/592f832f6e5d2d31f749392e48e8401b7625dec668d3d365d8d28f2b6c30/snakemake_interface_common-1.23.0.tar.gz", hash = "sha256:6ed14531a461417659364a0dd0acc51b786af4e26fc15cc5e00ff3d9fcaffacc", size = 13960, upload-time = "2026-03-08T21:54:29.251Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/90/a4a3652e832667917383f8d34112bdb37b78a6d14efc049e22a2c8192524/snakemake_interface_common-1.23.0-py3-none-any.whl", hash = "sha256:43bb10102d07149f27787dfc65a7342b86ff772973e7f9ca4940c89cb412a11e", size = 17162, upload-time = "2026-03-08T21:54:28.074Z" }, -] - -[[package]] -name = "snakemake-interface-executor-plugins" -version = "9.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass" }, - { name = "snakemake-interface-common" }, - { name = "throttler" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/50/de06b284c45a8e94fb8e4a12d5235065e78b49b8f84329dc10fe39f4b7dd/snakemake_interface_executor_plugins-9.4.0.tar.gz", hash = "sha256:9d4138897beacbaadaedad94b63f948eaeb604b7fc78f9cf65ac57f090f2c066", size = 16549, upload-time = "2026-03-08T17:04:02.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/2c/4422eb14884bb4955ce65819799ff6452dcd3aa5f8e158540a287877bbcf/snakemake_interface_executor_plugins-9.4.0-py3-none-any.whl", hash = "sha256:d335c093185897e4e3654a39bab3a6f1c57dc1a54f998c92bd2d88640ab4a05e", size = 22567, upload-time = "2026-03-08T17:04:01.296Z" }, -] - -[[package]] -name = "snakemake-interface-logger-plugins" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/0c/3fa5d592663c65669a867526604aadc6fbc235fb9284e94b49c0ef59aa41/snakemake_interface_logger_plugins-2.1.0.tar.gz", hash = "sha256:c89a00d2a398490cecd91b6dc6db8049cba93712d82e1d8f3000f3040bf3791c", size = 15917, upload-time = "2026-05-20T15:12:35.259Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/43/cfdaf562303012024c8f48efebdfe258eef3d895665bd7e0000fdeb2411d/snakemake_interface_logger_plugins-2.1.0-py3-none-any.whl", hash = "sha256:601e3e0770dd1b460c1dcb7bd33f4fc2eb8af5016634b80d985cb993f674ef46", size = 14313, upload-time = "2026-05-20T15:12:34.186Z" }, -] - -[[package]] -name = "snakemake-interface-report-plugins" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/d6/6160ed98de665d6871dd356597dbf726688cc786e88668359ca37b7d9f54/snakemake_interface_report_plugins-1.3.0.tar.gz", hash = "sha256:fc9495298bec4e69721ab8afe6d6d88a86966fda2eeb003db56b9a88b86d5934", size = 4283, upload-time = "2025-10-31T10:52:36.55Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/f0/df73f6abc9b5910e43612ae28c7b6f666af80c4edd46a216ef47599ab6cb/snakemake_interface_report_plugins-1.3.0-py3-none-any.whl", hash = "sha256:78da3931f70e79eef51e5645a40b172929e555fe4d86ff45d6b856e521a379db", size = 7251, upload-time = "2025-10-31T10:52:35.474Z" }, -] - -[[package]] -name = "snakemake-interface-scheduler-plugins" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "snakemake-interface-common" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/d9/d480807d2cfc2d132bc760d877d45ec8fbe620a24200ec4d2697c4a26031/snakemake_interface_scheduler_plugins-2.0.2.tar.gz", hash = "sha256:2797e8fa9019d983132c2b403f14d6fcd3c5ad4c8d8a66b984b4740a71cacc46", size = 8642, upload-time = "2025-10-20T13:58:12.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/d0/f4e9894c8aaf37efe3bf1afe15ee3cf0546d82b2713a589e266ee47bf2ef/snakemake_interface_scheduler_plugins-2.0.2-py3-none-any.whl", hash = "sha256:b9ddfa508bd480711de1770dfb24f3b813cfa3cd0f862f0127ef721ae5346915", size = 10766, upload-time = "2025-10-20T13:58:11.898Z" }, -] - -[[package]] -name = "snakemake-interface-storage-plugins" -version = "4.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "humanfriendly" }, - { name = "snakemake-interface-common" }, - { name = "tenacity" }, - { name = "throttler" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/6e/f3c5b2d621fd6a6b78d8cfc01fef6b926fe2c277f5ed77c5e4deeacb94eb/snakemake_interface_storage_plugins-4.4.1.tar.gz", hash = "sha256:b2b5bf05318af36955ebf2ce76c921c0fb06904ca98fb30e1657d88b7b7b6945", size = 14924, upload-time = "2026-03-16T11:16:01.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/49/1f72d8c05d8588f06a61f1af3deb125a4ff8169d2a41f1b839d62c824d1c/snakemake_interface_storage_plugins-4.4.1-py3-none-any.whl", hash = "sha256:6e075f448543cc3d0cb1ee5e6e5a83af9761e7085f3ce0922011685d098f231e", size = 18256, upload-time = "2026-03-16T11:15:59.582Z" }, -] - [[package]] name = "snowballstemmer" version = "3.1.1" @@ -3903,7 +3619,6 @@ workflow = [ { name = "cosmosis" }, { name = "fast-pt" }, { name = "mpi4py" }, - { name = "snakemake" }, ] [package.metadata] @@ -3957,7 +3672,6 @@ requires-dist = [ { name = "shear-psf-leakage", git = "https://github.com/CosmoStat/shear_psf_leakage.git?rev=develop" }, { name = "skyproj" }, { name = "smokescreen", git = "https://github.com/UNIONS-WL/Smokescreen?rev=main" }, - { name = "snakemake", marker = "extra == 'workflow'" }, { name = "sp-validation", extras = ["test", "docs"], marker = "extra == 'develop'" }, { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0" }, { name = "sphinxawesome-theme", marker = "extra == 'docs'", specifier = ">=5.3,!=6.0.3" }, @@ -4078,48 +3792,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.51" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/1c/71ee0f8a6b9d7316a1ccd30430b4c62b6c2e36adc96017a4e3a72dce49d6/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e", size = 3343835, upload-time = "2026-06-15T16:19:42.613Z" }, - { url = "https://files.pythonhosted.org/packages/2b/7c/7ab9f9aadc5944fdd06612484ed7918fe376ad871a5f50404dc1536e0194/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9", size = 3358470, upload-time = "2026-06-15T16:26:38.011Z" }, - { url = "https://files.pythonhosted.org/packages/d0/7d/ff77169fee6186de145a7f2b87006c39638391130abbab2b1f63ac6ea583/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389", size = 3289874, upload-time = "2026-06-15T16:19:45.212Z" }, - { url = "https://files.pythonhosted.org/packages/6f/3b/6c505903710d781b55bc3141ee34a062bf9745a6b5bc7333305b9ed63b33/sqlalchemy-2.0.51-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d", size = 3321692, upload-time = "2026-06-15T16:26:39.747Z" }, - { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, - { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, - { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, - { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, - { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, - { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, - { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, -] - -[[package]] -name = "sqlmodel" -version = "0.0.37" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/26/1d2faa0fd5a765267f49751de533adac6b9ff9366c7c6e7692df4f32230f/sqlmodel-0.0.37.tar.gz", hash = "sha256:d2c19327175794faf50b1ee31cc966764f55b1dedefc046450bc5741a3d68352", size = 85527, upload-time = "2026-02-21T16:39:47.038Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/e1/7c8d18e737433f3b5bbe27b56a9072a9fcb36342b48f1bef34b6da1d61f2/sqlmodel-0.0.37-py3-none-any.whl", hash = "sha256:2137a4045ef3fd66a917a7717ada959a1ceb3630d95e1f6aaab39dd2c0aef278", size = 27224, upload-time = "2026-02-21T16:39:47.781Z" }, -] - [[package]] name = "stack-data" version = "0.6.3" @@ -4234,24 +3906,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/68/025307ebd3c54c65e540d2a4ced11d0e5aa370c8ddb6d0e83066e5a39054/tables_io-1.1.2-py3-none-any.whl", hash = "sha256:4361e04dfcc10471bf9a3940d9ee6a1f903ac76ece51d32ef7fec7ea572b20c0", size = 39206, upload-time = "2026-06-18T16:21:19.577Z" }, ] -[[package]] -name = "tabulate" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, -] - -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - [[package]] name = "terminado" version = "0.18.1" @@ -4274,15 +3928,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] -[[package]] -name = "throttler" -version = "1.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/3f/47baf510c31e0e52ac0d80d9071e5e166ca069167fee4a6c13841f9d5f5f/throttler-1.2.3.tar.gz", hash = "sha256:d2f5b0b499d62f1fc984dcac8043450b606549b0097753a9c8a707f7427c27e1", size = 11792, upload-time = "2026-01-27T00:48:11.552Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/70/42d8796acc57c8bcd9ae395b1a6a0bbc833f738492a8ed192a44ccd58035/throttler-1.2.3-py3-none-any.whl", hash = "sha256:241ea3e97438dec4dc2f31ddc56dbd96262787a9b1d0598adfcc0bada1134b66", size = 9704, upload-time = "2026-01-27T00:48:09.544Z" }, -] - [[package]] name = "tifffile" version = "2026.6.1" @@ -4550,20 +4195,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] -[[package]] -name = "yte" -version = "1.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argparse-dataclass" }, - { name = "dpath" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/f5/7e44620e6e077bfe624b9a17c329b8e0d0159e176e1f1a93c2790428ab2c/yte-1.9.4.tar.gz", hash = "sha256:86a47e6d722cec9419a7ac88be57d0d6c4ce28f02860393b71a66f2c674069f6", size = 8101, upload-time = "2025-11-27T12:55:00.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/63/6a44729fdc60eb255a7b156a84e7552290174a9bf151e3b6c18e83d6fbfa/yte-1.9.4-py3-none-any.whl", hash = "sha256:5dac63303d3e6bc2ebadc36ece3c3fb09343772fe6e25e9356d9baf8f9dfaf6d", size = 10618, upload-time = "2025-11-27T12:55:01.685Z" }, -] - [[package]] name = "zarr" version = "3.2.1" diff --git a/workflow/README.md b/workflow/README.md index 4171074e..21d0419b 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -61,9 +61,14 @@ override, e.g. the image-sims `SIF`). A few rules shell out to a host toolchain (CosmoCov, ImageMagick) and keep `container: None`; each says why in its own docstring. -`OMP_NUM_THREADS` is not set by the profile either: the slurm executor's -`--export=ALL` propagates the driver's env, not a profile flag, so a rule that -needs it pinned sets it itself. Per-rule `mem_mb` / `runtime` stay on the rules. +The slurm executor submits with `--export=ALL`, so every job starts with the +launching shell's environment. `OMP_NUM_THREADS` is therefore not a profile +setting: a rule that needs it pinned sets it itself. A path in that +environment reaches nodes where it may not exist. For Snakemake's own cache +this is handled (the launch drops `XDG_CACHE_HOME`, and the profile keeps the +source cache off the shared filesystem), so a login shell that points it at +`/scratch` is fine; keep any other path you export on a shared disk. Per-rule +`mem_mb` / `runtime` stay on the rules. ### Off candide — the default profile @@ -93,6 +98,11 @@ This is the default because the alternative is incoherent: Snakemake's rule executes new script code against an old `import sp_validation` — the two halves of one commit, split. +The catalogue config is the launched checkout's `cosmo_val/cat_config.yaml`. +`COSMO_VAL` defaults to the launched checkout's `cosmo_val/output`, so writing +into another checkout's products means naming it; `COSMO_INFERENCE` defaults to +the shared candide tree. + **Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not `src/`. Editing a module under `src/` does not by itself mark outputs stale — force with `-F` or `--forcerun `. @@ -123,25 +133,25 @@ already uses the plain form; keep new paths the same. ### Run Snakemake from the host, never from inside the container -`snakemake` is a thin host-side tool, pinned once per machine: +`snakemake` is a thin host-side tool, installed once per machine on the +image's Python: ```bash -uv tool install snakemake==9.23.1 --with snakemake-executor-plugin-slurm +uv tool install --python 3.12 snakemake --with snakemake-executor-plugin-slurm ``` -(match the version to `snakemake` in this repo's `uv.lock`). Run every -`snakemake` command directly on the host — do not `apptainer shell` first. +Any Snakemake version works, but it must run on the image's Python (3.12): a +`script:` job loads the host's `snakemake` package into the image's interpreter, +since the image carries none. + +Run every `snakemake` command directly on the host — do not `apptainer shell` +first. Snakemake itself never touches the science stack; it only reads rule definitions and submits jobs. Each job carries its own `apptainer exec` wrapping from the profile (see above), so the container is where the science code runs, not where the orchestrator runs — one container per job, never a nested one. -Check for a stray `~/.local/bin/snakemake` (any host-side `pip install --user -snakemake` leaves one): Apptainer passes your `PATH` and mounts your `$HOME` by -default, so it can silently shadow the one `uv tool install` set up. `which -snakemake` should resolve under `uv tool dir`, not `~/.local/bin`. - ### The container image — one per person Everything runs one image, published by CI as a registry tag: @@ -181,7 +191,7 @@ job either gets the whole old image or the whole new one; jobs already running hold the old file open and finish against it unharmed. ```bash -salloc -p comp -c 4 --time=01:00:00 --exclude=n17,n09,n36 --no-shell # note the job id +salloc -p comp -c 4 --time=01:00:00 --exclude=n17,n36 --no-shell # note the job id srun --jobid= spv-container pull scancel ``` @@ -267,11 +277,20 @@ For the image-sims workflow, set `image_sims: {sif: ...}` in your run config. Either way the image has to sit under one of the profile's bind mounts to be visible. -One trap to know: the `script:` directive bind-mounts the host orchestrator's -`snakemake` into the job and *appends* it to `sys.path`, so a `snakemake` -importable inside the image wins the lookup. If `script:` rules start failing -with `ModuleNotFoundError: No module named 'snakemake.iocontainers'` or similar, -an in-image snakemake older than the host's is the first thing to check. +### Checking the workflow itself + +`workflow/tests/` checks DAG properties through the host launcher, on a toy +checkout and — on candide — on the real papers: + +```bash +uv run --isolated --no-project --python 3.12 --with snakemake \ + --with snakemake-executor-plugin-slurm --with pytest \ + pytest workflow/tests +``` + +CI runs the same suite with `-m "not candide"`. `test_container_smoke` +submits one real SLURM job through the candide profile, so it runs only where +`sbatch` exists — a candide login node — and skips on compute nodes. ### `snakemake` in `script:` files diff --git a/workflow/common.py b/workflow/common.py index 9927a6f6..ede56f55 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -1,4 +1,9 @@ -"""Shared helpers for the B-modes Snakemake workflow.""" +"""Shared helpers for every Snakefile that composes workflow/. + +Imported by the host Snakemake, where sp_validation is not installed: this +module imports only the standard library and snakemake, and loads the +stdlib-only project modules it needs by file path. +""" import importlib.util import json @@ -7,17 +12,13 @@ import sys from pathlib import Path -# This checkout's importable source tree: workflow/common.py -> /src. -REPO_SRC = Path(__file__).resolve().parent.parent / "src" +# The checkout this workflow was launched from: workflow/common.py -> . +REPO_ROOT = Path(__file__).resolve().parent.parent +REPO_SRC = REPO_ROOT / "src" # The container model lives in the package (``sp_validation/container.py``). # Taken from *this checkout's* src/, so the workflow and the ``spv-container`` # CLI can never disagree about which image to run. -# -# Loaded by file path rather than as ``sp_validation.container``: snakemake runs -# on the host, where sp_validation is usually not installed, and importing the -# package would drag in ``__init__`` -> ``version`` -> a metadata warning on -# every launch. The module itself is stdlib-only, so this costs nothing. _container = importlib.util.module_from_spec( importlib.util.spec_from_file_location( "_spv_container", REPO_SRC / "sp_validation" / "container.py" @@ -30,20 +31,27 @@ image_revision = _container.image_revision resolve_image = _container.resolve_image +# Every job inherits this launch's environment (the slurm executor submits with +# --export=ALL), and the Snakemake each job step starts keeps its source cache +# under XDG_CACHE_HOME, which a login shell may point at node-local storage. +# Without it, jobs use the home directory's cache, which every node mounts. +os.environ.pop("XDG_CACHE_HOME", None) + # Output roots are env-overridable so a reproduction run can write into a -# fresh tree without clobbering (or silently reusing) prior products. -COSMO_VAL = Path( - os.environ.get( - "COSMO_VAL", "/n17data/cdaley/unions/code/sp_validation/cosmo_val/output" - ) -) +# fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL +# defaults to the launched checkout's own (gitignored) cosmo_val/output, so a +# launch writes into another checkout's products only when it names that tree; +# COSMO_INFERENCE defaults to the shared tree on candide. +COSMO_VAL = Path(os.environ.get("COSMO_VAL", REPO_ROOT / "cosmo_val" / "output")) COSMO_INFERENCE = Path( os.environ.get( "COSMO_INFERENCE", "/n17data/cdaley/unions/code/sp_validation/cosmo_inference" ) ) -CAT_CONFIG = "/n17data/cdaley/unions/code/sp_validation/cosmo_val/cat_config.yaml" +# The catalogue config of the launched checkout: the one file both the host +# (CATALOG_CONFIG, loaded in configure) and every job read catalogues from. +CAT_CONFIG = str(REPO_ROOT / "cosmo_val" / "cat_config.yaml") # "blind" is the glass-mock A/B/C realisation convention, NOT Smokescreen # blinding (a separate axis: the concealed=True SACC stamp). The name is baked # into on-disk filenames we do not own (e.g. nz_{version}_{A|B|C}.txt). @@ -118,9 +126,7 @@ def resolve_container(override=None): ``resolve_image()``, so jobs run what interactive ``spv-container`` work runs. """ - if override: - return str(override) - return resolve_image()[0] + return str(override) if override else resolve_image()[0] def warn_if_image_stale(): @@ -161,9 +167,11 @@ def warn_if_image_stale(): def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + from snakemake.common.configfile import load_configfile + inject_checkout_pythonpath(workflow_config) warn_if_image_stale() - CATALOG_CONFIG = workflow_config + CATALOG_CONFIG = load_configfile(CAT_CONFIG) FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" @@ -287,17 +295,10 @@ def build_redshift_path(version, blind): # --------------------------------------------------------------------------- # ξ± angular grids # --------------------------------------------------------------------------- -# A grid is a binning plus how its covariance is estimated: (min_sep, max_sep, -# nbins, npatch, cov). `reporting` is the analysis grid, `integration` the fine -# one the B-mode integrals run over, `cosebis` the fine patched grid COSEBIs -# propagates its covariance from. cov is "jackknife" (dense, from the patches), -# "diagonal" (TreeCorr varxip/varxim) or "none". -XI_KEYS = ( - "min_sep", - "max_sep", - "nbins", - "npatch", -) # the binning; cov is not in the name +# A grid is a binning: (min_sep, max_sep, nbins, npatch). `reporting` is the +# analysis grid, `integration` the fine one both B-mode statistics (COSEBIs, +# pure-E/B) integrate over. +XI_KEYS = ("min_sep", "max_sep", "nbins", "npatch") def xi_grids(config, fiducial): @@ -332,22 +333,11 @@ def xi_grids(config, fiducial): ), } grids["integration"].setdefault("npatch", 1) - cb = cv.get("cosebis") - if cb: - grids["cosebis"] = { - "min_sep": cb["min_sep_int"], - "max_sep": cb["max_sep_int"], - "nbins": cb["nbins_int"], - "npatch": cb["npatch"], - } for grid in grids.values(): for key in ("min_sep", "max_sep"): grid[key] = float(grid[key]) for key in ("nbins", "npatch"): grid[key] = int(grid[key]) - # A jackknife estimate needs patches; at npatch=1 TreeCorr's var_method - # is "shot" and the diagonal is all it can offer. - grid.setdefault("cov", "jackknife" if grid["npatch"] > 1 else "none") return grids @@ -397,19 +387,10 @@ def get_shear_catalog(wildcards): # turns each diagnostic into a rule keyed on the real data products it writes # under COSMO_VAL. Where a method only emits a figure (no data product), the # rule declares a sentinel under CV_SENTINELS so the DAG stays trackable. -# -# COSMO_VAL is the cosmo_val/output directory (already defined above), the same -# location every `cv.*` method writes to via `cc["paths"]["output"]`. # Sentinel directory for pure-plot leaf rules (no natural data-product output). CV_SENTINELS = COSMO_VAL / "snakemake_sentinels" -# Working directory in which `CosmologyValidation` is instantiated: its -# catalogue config comes explicitly from CAT_CONFIG, and it writes to -# `./output` unless COSMO_VAL is set. Resolved to the live (non-worktree) -# checkout so rules share the output tree with interactive runs. -CV_RUNDIR = "/n17data/cdaley/unions/code/sp_validation/cosmo_val" - def cv_basename(version, fiducial=None): """Reproduce CosmologyValidation.basename() for a version. @@ -461,17 +442,17 @@ def cv_basename(version, fiducial=None): ) -def cv_init_params(config, version_list=None): +def cv_init_params(config): """Assemble the CosmologyValidation(...) constructor kwargs from config. Centralizes the run-specific instantiation so every cosmo_val rule script - builds an identical `cv`. The catalogue config is always CAT_CONFIG, never - the constructor's cwd-relative default. `version_list` overrides - config["versions"] (used by per-version rules that pass a single version). + builds an identical `cv`: catalogues from CAT_CONFIG and products under + COSMO_VAL, never the constructor's cwd-relative defaults. """ cv = config["cosmo_val"] return dict( - versions=version_list if version_list is not None else config["versions"], + versions=config["versions"], catalog_config=CAT_CONFIG, + output_dir=str(COSMO_VAL), **{key: cv[key] for key in CV_INIT_KEYS}, ) diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 19afd89a..39649717 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -1,68 +1,72 @@ -# Committed SLURM profile for the candide cluster (IAP). -# -# Drive any target with +# SLURM profile for the candide cluster (IAP). Drive any target with # # snakemake --profile workflow/profiles/candide \ # -s workflow/image_sims/Snakefile \ # --configfile # -# Snakemake owns scheduling and the container wrapping; run it host-side, never -# inside an ``apptainer shell``. workflow/README.md is the full story. +# from the host, never inside an ``apptainer shell``; workflow/README.md is the +# full story. executor: slurm -# --- GENERIC: mirrored in workflow/profiles/default/config.yaml ------------- +# --- mirrored in workflow/profiles/default/config.yaml ---------------------- software-deployment-method: apptainer -# Rerun a job when its code / params / inputs change, not only on mtime. -# Caveat: "code" watches rule bodies and ``script:`` files, not ``src/`` -- -# editing a module under src/ does not mark outputs stale on its own. +# Every trigger but software-env, which hashes the image *path*: that path is +# per person (~/.cache/sp_validation/...) and switches between SIF and sandbox, +# so it would rerun everything without the stack having changed. +# "code" watches rule bodies and ``script:`` files, not ``src/``. rerun-triggers: ["mtime", "params", "input", "code"] -# Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. -latency-wait: 5 -# --- end GENERIC ------------------------------------------------------------ +# A job's write can take more than 5 s to show on the launching host (candide's +# /home does); a shorter wait reads a finished job as failed and re-runs it. +latency-wait: 60 +# --- end mirrored ----------------------------------------------------------- -# No ``apptainer-prefix``: the entry Snakefiles resolve ``container:`` to this -# user's own image path, so there is nothing for Snakemake to cache. -# -# candide's disks, plus the one machine-specific env var: the host OpenMPI libs -# MPI rules need to find libmpi inside the container (only rules importing -# mpi4py care; harmless for the rest). The bind list matches ``spv-container -# exec``'s default; keep the two in step. +# candide's disks; the same list as ``spv-container exec``'s default. apptainer-args: >- --cleanenv --bind /home,/scratch,/automnt,/n17data,/n23data1,/n09data - --env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib -# Cluster policy applied to every job unless a rule overrides it. Excludes are -# the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, n36). +# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``): Snakemake reads a +# bare number here as SECONDS. (In a rule's own ``resources:`` a bare +# integer is minutes.) 60m is the partitions' default; setting it keeps the +# slurm executor from warning on every job that names none. # -# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's -# resource parser reads a bare number as SECONDS, so ``runtime: 60`` would -# silently give every job a 60-second wall clock and kill it on start. -# (A bare integer in a rule's own ``resources: runtime=720`` is fine -- -# rule-level numeric runtime is read as minutes; the seconds trap is only -# the CLI/default-resources parser.) +# * ``cpus_per_task`` is 12 to CAP JOBS PER NODE, not because a job needs 12 +# cores: past ~4 concurrent jobs on a 48-core node, candide's per-user +# process limit (``ulimit -u 1200``) crashes apptainer ("can't start new +# thread"). After launching a fan-out, ``squeue -u $USER -o "%C %l"`` +# should show 12 CPUs and the intended wall clock. # -# * ``cpus_per_task`` is pinned to 12 to CAP JOBS PER NODE, not because a job -# needs 12 cores. candide's per-user process limit is ``ulimit -u 1200`` -# per node, and apptainer crashes ("can't start new thread") beyond ~4 -# concurrent jobs on a 48-core node; 12 CPUs/job holds SLURM to ~4 jobs per -# node. +# * ``slurm_account``: the slurm executor's guess fails on candide (its +# ``sacct`` call errors) and warns on every launch. # -# * After launching a real fan-out, verify the request landed: -# ``squeue -u $USER -o "%C %l"`` should show 12 (CPUs) and the intended -# wall clock. +# * The excludes are the flaky nodes: n17 (mount issues), n36. default-resources: slurm_account: "cusers" slurm_partition: "comp,pscomp" runtime: "60m" cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n09,n36'" + slurm_extra: "'--exclude=n17,n36'" + +# Snakemake refuses a real run on a remote executor without a job bound. +jobs: 100 -# Retry a job once on transient node failure, and keep the SLURM logs of -# successful jobs (candide debugging). +# Jobs inherit the launch env; leaving out source-cache keeps them from being +# handed the launch's (possibly node-local) cache path. common.py drops +# XDG_CACHE_HOME for the jobstep's own Snakemake. +shared-fs-usage: + - persistence + - input-output + - software-deployment + - sources + - storage-local-copies + - software-deployment-cache + +# One retry absorbs a node failure in a large fan-out, which would otherwise +# stop the launch scheduling new jobs. retries: 1 + +# A successful job's SLURM log is the only record of what its script printed. slurm-keep-successful-logs: true diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml index bf272c03..419b88a4 100644 --- a/workflow/profiles/default/config.yaml +++ b/workflow/profiles/default/config.yaml @@ -1,39 +1,29 @@ -# Machine-independent profile: the container model, and nothing else. -# -# Use it anywhere that is not candide -- a laptop, a workstation, another -# cluster's interactive node: +# Machine-independent profile: the container model, and nothing else. Use it +# anywhere that is not candide (a laptop, a workstation, another cluster): # # snakemake --profile workflow/profiles/default -s workflow/Snakefile \ # --configfile -j 4 # -# On candide -- where the analysis actually runs -- use -# `--profile workflow/profiles/candide` instead: the GENERIC block below plus -# the SLURM executor and candide's machine layer. -# -# Requirements are the same everywhere: `apptainer` on PATH, and `snakemake` -# installed host-side (`uv tool install ...`, see workflow/README.md) -- never -# run from inside an apptainer shell. +# On candide use workflow/profiles/candide. Either way, `apptainer` on PATH and +# `snakemake` installed host-side (workflow/README.md); never run from inside +# an apptainer shell. -# --- GENERIC: mirrored in workflow/profiles/candide/config.yaml ------------- -# Wrap each job's `shell:`/`script:` command in `apptainer exec`, using the -# image named by the entry Snakefile's `container:` directive. +# --- mirrored in workflow/profiles/candide/config.yaml ---------------------- software-deployment-method: apptainer -# Rerun a job when its code / params / inputs change, not only on mtime. -# Caveat: "code" watches rule bodies and `script:` files, not `src/` -- editing -# a module under src/ does not mark outputs stale on its own. +# Every trigger but software-env, which hashes the image *path*: that path is +# per person (~/.cache/sp_validation/...) and switches between SIF and sandbox, +# so it would rerun everything without the stack having changed. +# "code" watches rule bodies and `script:` files, not `src/`. rerun-triggers: ["mtime", "params", "input", "code"] -# Give an appearing output file a moment on networked filesystems before -# Snakemake calls a job failed for a missing output. -latency-wait: 5 -# --- end GENERIC ------------------------------------------------------------ +# A job's write can take more than 5 s to show on the launching host (candide's +# /home does); a shorter wait reads a finished job as failed and re-runs it. +latency-wait: 60 +# --- end mirrored ----------------------------------------------------------- -# Binds are the one thing you almost certainly need to edit for your machine: -# whatever paths your inputs, outputs and checkout live under. `--cleanenv` so a -# job's environment is the image's, not your shell's. If $HOME and the working -# directory cover everything (apptainer mounts both by default), drop `--bind`. +# `--cleanenv` so a job's environment is the image's, not your shell's. +# Snakemake runs each job with `--home `, so apptainer mounts +# that directory and not your home: bind /home, and add `--bind` for any other +# disk your inputs, outputs or checkout live on. apptainer-args: "--cleanenv --bind /home" - -# No `apptainer-prefix`, here or on candide: the entry Snakefiles resolve -# `container:` to this user's own image path (workflow/README.md). diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 87138bc0..34735a85 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -6,14 +6,14 @@ # each diagnostic is a rule, and the rules are linked by the SACC parts and # products they write under COSMO_VAL (= cosmo_val/output): # -# catalogue ──→ xi (one job per grid: reporting, integration, cosebis) -# │ -# ├─ reporting part ──┬─→ pure_eb (part, npz, figures) -# ├─ integration part ┘ │ -# ├─ cosebis part ─────→ cosebis (part, npz, figures) +# catalogue ──→ xi (one job per grid: reporting, integration) +# ├─ reporting part ──────→ pure_eb (part, npz, figures) +# ├─ integration part ─┬──→ pure_eb +# │ └──→ cosebis (part, npz, figures) # └─ reporting .txt ──→ 2pcf plot, ratio_xi_sys_xi +# CosmoCov ξ±, integration grid ──→ pure_eb, cosebis (their covariances) # catalogue ──→ pseudo_cl (part) ──┬─→ pseudo-Cl figures -# CosmoCov ──→ covariance ─────────┤ +# CosmoCov ξ±, reporting grid ─────┤ # rho/tau (part + FITS) ───────────┼─→ summarize_bmodes (reads the products) # └─→ assemble_sacc ──→ {version}.sacc # @@ -73,7 +73,7 @@ def cv_tau_stats(version): def _pure_eb_stub(version): """Shared stem of the pure-E/B diagnostic products (npz + figures).""" - eb = CV["integration"] + eb = XI_GRIDS["integration"] return str( COSMO_VAL / ( @@ -101,39 +101,44 @@ def cv_pure_eb_figures(version): } -def cv_xi_cov_integration(version): - """CosmoCov gaussian ξ± covariance on the integration grid. - - The covariance model the pure-E/B Monte Carlo draws from; gaussian because - the draws only need the scatter a Gaussian field would give. - """ - integ = CV["integration"] +def _grid_cov(version, grid, gaussian): + """CosmoCov-processed ξ± covariance on a named grid's own binning.""" + binning = XI_GRIDS[grid] return covariance_path( version, FIDUCIAL["blind"], - gaussian="g", - min_sep=integ["min_sep"], - max_sep=integ["max_sep"], - nbins=integ["nbins"], + gaussian=gaussian, + min_sep=binning["min_sep"], + max_sep=binning["max_sep"], + nbins=binning["nbins"], mask_suffix=DEFAULT_MASK_SUFFIX, ) +def cv_xi_cov_integration(version): + """CosmoCov gaussian ξ± covariance on the integration grid. + + The one analytic covariance both B-mode statistics take: the pure-E/B Monte + Carlo draws from it and COSEBIs carry it through their kernel. Gaussian + because B-modes only need the scatter a Gaussian field would give. + """ + return _grid_cov(version, "integration", "g") + + def _cosebis_stub(version): """Shared stem of the COSEBIs diagnostic products (npz + figures). - varmethod names where the covariance came from, and these products are the - propagated one — which also keeps them clear of the paths plot_cosebis - builds for its own byproducts, so nothing overwrites a declared output. + Named by the grid, where the covariance came from (varmethod), the mode + count and the fiducial scale cut. """ - cb = CV["cosebis"] + integ = XI_GRIDS["integration"] fsc = CV["fiducial_scale_cut"] return str( COSMO_VAL / ( - f"{version}_cosebis_minsep={cb['min_sep_int']}" - f"_maxsep={cb['max_sep_int']}_nbins={cb['nbins_int']}" - f"_npatch={cb['npatch']}_varmethod=propagated_nmodes={cb['nmodes']}" + f"{version}_cosebis_minsep={integ['min_sep']}" + f"_maxsep={integ['max_sep']}_nbins={integ['nbins']}" + f"_varmethod=analytic_nmodes={CV['cosebis']['nmodes']}" f"_scalecut={fsc[0]}-{fsc[1]}" ) ) @@ -168,16 +173,8 @@ def cv_pseudo_cl_cov(version): def cv_xi_cov(version): - """CosmoCov-processed ξ± covariance, on the reporting grid's own binning.""" - return covariance_path( - version, - FIDUCIAL["blind"], - gaussian="ng", - min_sep=CV["theta_min"], - max_sep=CV["theta_max"], - nbins=CV["nbins"], - mask_suffix=DEFAULT_MASK_SUFFIX, - ) + """CosmoCov ξ± covariance on the reporting grid, the terminal file's.""" + return _grid_cov(version, "reporting", "ng") def cv_cosebis_sacc(version): @@ -207,13 +204,8 @@ def cv_analysis_sacc(version): return str(COSMO_VAL / f"{version}.sacc") -# Common params block shared by every cosmo_val rule: the cv constructor kwargs -# plus the run directory the object must be instantiated in. -def cv_params(version_list=None): - return dict( - cv_init=cv_init_params(config, version_list=version_list), - rundir=CV_RUNDIR, - ) +# The CosmologyValidation constructor kwargs every cv_runner rule passes. +CV_INIT = cv_init_params(config) # --------------------------------------------------------------------------- @@ -234,7 +226,7 @@ rule cv_plot_rho_stats: output: sentinel=str(CV_SENTINELS / "plot_rho_stats.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: runtime=20, script: @@ -248,7 +240,7 @@ rule cv_plot_tau_stats: output: sentinel=str(CV_SENTINELS / "plot_tau_stats.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: runtime=20, script: @@ -263,7 +255,7 @@ rule cv_rho_tau_fits: output: sentinel=str(CV_SENTINELS / "rho_tau_fits.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=120, @@ -280,7 +272,7 @@ rule cv_footprints: output: sentinel=str(CV_SENTINELS / "footprints.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=60, @@ -293,7 +285,7 @@ rule cv_objectwise_leakage: output: sentinel=str(CV_SENTINELS / "objectwise_leakage.done"), params: - **cv_params(), + cv_init=CV_INIT, threads: 12 resources: mem_mb=30000, @@ -307,7 +299,7 @@ rule cv_weights: output: weight_hist=str(COSMO_VAL / "weight_hist.png"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=30, @@ -329,7 +321,7 @@ rule cv_additive_bias: output: additive_bias=str(COSMO_VAL / "additive_bias.json"), params: - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=30, @@ -344,7 +336,7 @@ rule cv_plot_2pcf: output: sentinel=str(CV_SENTINELS / "plot_2pcf.done"), params: - **cv_params(), + cv_init=CV_INIT, resources: runtime=20, script: @@ -361,7 +353,7 @@ rule cv_ratio_xi_sys_xi: ratio=str(COSMO_VAL / "ratio_xi_sys_xi.png"), params: offset=0.1, - **cv_params(), + cv_init=CV_INIT, resources: mem_mb=16000, runtime=120, @@ -393,7 +385,6 @@ rule cv_plot_pseudo_cl: # Style is per catalogue, so the derived variants take their parent's. markers=[CATALOG_CONFIG[base_version(v)]["marker"] for v in CV_VERSIONS], colours=[CATALOG_CONFIG[base_version(v)]["colour"] for v in CV_VERSIONS], - rundir=CV_RUNDIR, resources: mem_mb=8000, runtime=20, @@ -427,7 +418,6 @@ rule cv_pure_eb: n_samples=CV.get("n_mc_samples", 1000), cosmo_params=CV["cosmo_params"], fiducial_scale_cut=CV["fiducial_scale_cut"], - rundir=CV_RUNDIR, threads: 24 resources: mem_mb=40000, @@ -437,26 +427,26 @@ rule cv_pure_eb: rule cv_cosebis: - """COSEBIs E/B decomposition for one version, from its ξ± part. + """COSEBIs E/B decomposition for one version, from its integration-grid part. - Values, covariance and PTEs all come from the part: the COSEBIs covariance - is the part's ξ± covariance through the same kernel as the modes. + The modes come from the part; the covariance is the CosmoCov ξ± covariance + on the same grid through the same kernel as the modes. """ input: - xi=lambda w: cv_xi_sacc(w.version, "cosebis"), + xi=lambda w: cv_xi_sacc(w.version, "integration"), + cov=lambda w: cv_xi_cov_integration(w.version), output: npz=cv_cosebis_npz("{version}"), sacc=cv_cosebis_sacc("{version}"), **cv_cosebis_figures("{version}"), params: version="{version}", - min_sep=CV["cosebis"]["min_sep_int"], - max_sep=CV["cosebis"]["max_sep_int"], - nbins=CV["cosebis"]["nbins_int"], + min_sep=XI_GRIDS["integration"]["min_sep"], + max_sep=XI_GRIDS["integration"]["max_sep"], + nbins=XI_GRIDS["integration"]["nbins"], nmodes=CV["cosebis"]["nmodes"], scale_cuts=CV["cosebis"]["scale_cuts"], fiducial_scale_cut=CV["fiducial_scale_cut"], - rundir=CV_RUNDIR, threads: 24 resources: mem_mb=48000, @@ -487,7 +477,6 @@ rule cv_summarize_bmodes: max_sep=CV["theta_max"], nbins=CV["nbins"], include_pseudo_cl=CV.get("include_pseudo_cl", False), - rundir=CV_RUNDIR, resources: mem_mb=8000, runtime=20, diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index e9185428..9c26b775 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -4,9 +4,9 @@ def get_cat_params(version): """Extract covariance parameters (area, n_e, sigma_e) from catalog config.""" base_version = version.replace("_leak_corr", "") - if base_version not in config: + if base_version not in CATALOG_CONFIG: raise KeyError(f"Catalog configuration not found for {base_version}") - cov_th = config[base_version]["cov_th"] + cov_th = CATALOG_CONFIG[base_version]["cov_th"] return cov_th["A"], cov_th["n_e"], cov_th["sigma_e"] diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 1b6663b6..4e8b75f6 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -13,16 +13,11 @@ def xi_binning(grid): return grid_binning(XI_GRIDS[grid]) -def xi_grid_of(wildcards): - """Grid label for the binning a job was requested with.""" - return grid_of(XI_GRIDS, {key: getattr(wildcards, key) for key in XI_KEYS}) - - rule xi: """TreeCorr ξ±(θ) for one version on one angular grid. One rule for every grid: outputs are named by their binning, so a request - binds the wildcards and `xi_grid_of` resolves the grid label from them. + binds the wildcards and the grid label resolves from them. """ input: catalog=get_shear_catalog, @@ -38,8 +33,7 @@ rule xi: npatch="{npatch}", cat_config=CAT_CONFIG, output_dir=str(COSMO_VAL), - grid=lambda w: xi_grid_of(w), - cov=lambda w: XI_GRIDS[xi_grid_of(w)]["cov"], + grid=lambda w: grid_of(XI_GRIDS, w), resources: # The fine integration grid needs more memory and wall time than the # ~20-bin reporting one; scale on nbins rather than splitting the rule. @@ -68,6 +62,7 @@ rule rho_tau_stats: resources: mem_mb=30000, disk_mb=20000, + runtime=360, script: "../scripts/run_rho_tau.py" diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index c9241eaa..d431b470 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -1,11 +1,12 @@ """Rule cv_cosebis: COSEBIs E/B decomposition for one version. -A consumer of the ξ± part alone — values, covariance, PTEs and figures all -derive from it, so nothing here touches a catalogue. The part's ξ± covariance -goes through the same linear kernel as the modes to give the COSEBIs -covariance; its ``npatch`` metadata sets the Hartlap debiasing. +A consumer of the integration-grid ξ± part and the CosmoCov ξ± covariance on +the same grid — nothing here touches a catalogue. The covariance goes through +the same linear kernel as the modes to give the COSEBIs covariance; it is +analytic, so no Hartlap debiasing applies. """ +import numpy as np from cv_runner import _unbuffer_streams, verify_outputs from sp_validation import sacc_io @@ -26,18 +27,17 @@ fiducial_scale_cut = tuple(p["fiducial_scale_cut"]) part = sacc_io.load(snakemake.input["xi"]) -theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="cosebis") +theta, xip, xim = sacc_io.get_xi(part, (0, 0), grid="integration") edges = log_bin_edges(p["min_sep"], p["max_sep"], p["nbins"]) results = cosebis_scan_from_xi( theta, xip, xim, - part.covariance.dense, + np.loadtxt(snakemake.input["cov"]), *edges, nmodes=p["nmodes"], scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], - npatch=part.metadata["npatch"], ) fiducial_key = find_conservative_scale_cut_key(results, fiducial_scale_cut) @@ -50,7 +50,7 @@ fiducial_scale_cut=fiducial_scale_cut, ) plot_cosebis_covariance_matrix( - fiducial, version, "jackknife", snakemake.output["figure_covariance"] + fiducial, version, "analytic", snakemake.output["figure_covariance"] ) plot_cosebis_scale_cut_heatmap( results, diff --git a/workflow/scripts/cv_runner.py b/workflow/scripts/cv_runner.py index 040cbe8a..601f2e14 100644 --- a/workflow/scripts/cv_runner.py +++ b/workflow/scripts/cv_runner.py @@ -28,15 +28,11 @@ def _unbuffer_streams(): def make_cv(snakemake): """Build a CosmologyValidation from a rule's ``snakemake.params``. - ``params["cv_init"]`` is the kwargs dict assembled by common.cv_init_params. - The catalogue config path arrives explicitly in ``cv_init``; the object is - created with the run directory as cwd so it writes under ``output/`` - exactly as interactive runs do. + ``params["cv_init"]`` is the kwargs dict assembled by common.cv_init_params, + which names the catalogue config and the output directory explicitly. """ from sp_validation.cosmo_val import CosmologyValidation - rundir = snakemake.params["rundir"] - os.chdir(rundir) return CosmologyValidation(**dict(snakemake.params["cv_init"])) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 9200bfaa..ec44142a 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -42,7 +42,7 @@ # this table wants. cosebis = np.load(snakemake.input["cosebis"][i]) row["COSEBIS"] = float(cosebis["pte_B"]) - cov_methods.add("COSEBIs: propagated from the ξ± covariance") + cov_methods.add("COSEBIs: analytic (CosmoCov ξ± through the COSEBIs kernel)") if p["include_pseudo_cl"]: from astropy.io import fits diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 1f69ee6d..28c8c521 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -16,12 +16,12 @@ grids are the same compute with different ``--min-sep/--max-sep/--nbins``. ``CosmologyValidation.calculate_2pcf`` writes the ``.txt`` dump (a raw byproduct); the ξ± data product is born as SACC here, a *part* named by its -binning and tagged with its ``--grid``. The part carries the covariance its -grid configures (``--cov``): the dense jackknife estimate from the patches, the -TreeCorr ``varxip``/``varxim`` diagonal, or none. +binning and tagged with its ``--grid``. The part carries the covariance the +measurement estimated: the dense jackknife covariance when it had patches, the +shot-noise ``varxip``/``varxim`` diagonal when it had none. -``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) -so lc can point each run at its own ``{output}`` tree. +``output_dir`` is passed explicitly so lc can point each run at its own +``{output}`` tree. """ import argparse @@ -44,7 +44,6 @@ def run_2pcf( output_dir, sacc_out=None, grid="reporting", - cov="none", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -77,10 +76,8 @@ def run_2pcf( nbins=nbins, ) - if cov == "jackknife" and int(npatch) < 2: - raise ValueError(f"cov='jackknife' needs patches; got npatch={npatch}") - # Born-as-SACC ξ± part. theta = meanr; theta_nom = rnom. + jackknife = gg.var_method == "jackknife" s = xi_to_sacc( cv.sacc_nz(ver), cv.sacc_metadata(ver), @@ -91,10 +88,8 @@ def run_2pcf( theta_nom=gg.rnom, npairs=gg.npairs, weight=gg.weight, - covariance=gg.cov if cov == "jackknife" else None, - variances=( - np.concatenate([gg.varxip, gg.varxim]) if cov == "diagonal" else None - ), + covariance=gg.cov if jackknife else None, + variances=None if jackknife else np.concatenate([gg.varxip, gg.varxim]), ) out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], @@ -116,7 +111,6 @@ def _from_snakemake(smk): cat_config=p["cat_config"], output_dir=p["output_dir"], grid=p.get("grid", "reporting"), - cov=p.get("cov", "none"), # The SACC part goes exactly where the rule declares it; the .txt # byproduct still lands under the resolved output dir. sacc_out=smk.output["sacc"], @@ -149,12 +143,6 @@ def _from_cli(argv=None): ap.add_argument( "--grid", default="reporting", help="SACC grid tag for the measured points" ) - ap.add_argument( - "--cov", - default="none", - choices=["jackknife", "diagonal", "none"], - help="Covariance the part carries", - ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -165,7 +153,6 @@ def _from_cli(argv=None): cat_config=a.cat_config, output_dir=a.out, grid=a.grid, - cov=a.cov, ) diff --git a/workflow/tests/conftest.py b/workflow/tests/conftest.py new file mode 100644 index 00000000..5540e091 --- /dev/null +++ b/workflow/tests/conftest.py @@ -0,0 +1,222 @@ +"""Host-side harness: the workflow as a person launches it. + +These tests run under the host launcher, never inside the image:: + + uv run --isolated --no-project --python 3.12 --with snakemake \\ + --with snakemake-executor-plugin-slurm --with pytest pytest workflow/tests + +``sp_validation`` is absent from that environment, so every Snakefile has to +parse with the standard library and Snakemake alone -- the condition a host +Snakemake is in. The ``candide`` tests need candide itself; CI deselects them. + +The ``toy`` fixture is a disposable checkout: copies of ``workflow/`` and +``papers/cosmo_val/``, this checkout's ``src/`` symlinked in, a one-catalogue +``cosmo_val/cat_config.yaml``, a touched catalogue file, the processed CosmoCov +covariances already in place (their inputs live on candide), and both output +roots in tmp. +""" + +import dataclasses +import importlib.util +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[2] + +# The toy catalogue and its leakage-corrected variant. +VERSIONS = ("SP_v0.1", "SP_v0.1_leak_corr") + + +@dataclasses.dataclass +class Job: + rule: str + input: list + output: list + wildcards: dict + + +_RULE = re.compile(r"^(?:local)?rule (\w+):$") +_FIELD = re.compile(r"^ (\w+): (.*)$") + + +def parse_jobs(text): + """The jobs a ``snakemake -n`` listing schedules.""" + jobs, fields = [], None + for line in text.splitlines(): + if match := _RULE.match(line): + fields = {"rule": match[1]} + jobs.append(fields) + elif fields is not None and (match := _FIELD.match(line)): + fields[match[1]] = match[2] + else: + fields = None + return [ + Job( + rule=f["rule"], + input=f["input"].split(", ") if "input" in f else [], + output=f["output"].split(", ") if "output" in f else [], + wildcards=dict( + pair.split("=", 1) + for pair in f.get("wildcards", "").split(", ") + if pair + ), + ) + for f in jobs + ] + + +def _load_module(path, name, env): + """Import a workflow module by path under ``env`` (it reads env at import).""" + saved = os.environ.copy() + os.environ.update(env) + try: + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + os.environ.clear() + os.environ.update(saved) + return module + + +# The package's image model, loaded by path: sp_validation is absent here. +container = _load_module( + REPO / "src" / "sp_validation" / "container.py", "spv_container", {} +) + + +@dataclasses.dataclass +class Toy: + root: Path + rundir: Path + cosmo_val: Path + cosmo_inference: Path + env: dict + config: dict + common: object + covariances: dict # (version, "g" | "ng") -> the processed CosmoCov file + + def snakemake(self, *args, cwd=None, env=None, timeout=300): + """Run the host Snakemake in the toy's paper directory, or in ``cwd``. + + ``env`` replaces the toy's environment. + """ + return subprocess.run( + [sys.executable, "-m", "snakemake", "--cores", "1", *args], + cwd=cwd or self.rundir, + env=env or self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + + +def _cat_config(catalogue): + entry = { + "subdir": str(catalogue.parent), + "pipeline": "SP", + "colour": "orange", + "marker": "^", + "cov_th": {"A": 100.0, "n_e": 5.0, "sigma_e": 0.3}, + "shear": { + "path": str(catalogue), + "redshift_path": str(catalogue.parent / "nz_SP_v0.1_A.txt"), + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_col_corrected": "e1_leak_corrected", + "e2_col_corrected": "e2_leak_corrected", + }, + } + return {VERSIONS[0]: entry, "paths": {"output": "./output"}} + + +@pytest.fixture(scope="session") +def toy(tmp_path_factory): + root = tmp_path_factory.mktemp("toy") + skip = shutil.ignore_patterns(".snakemake", "__pycache__", "tests") + shutil.copytree(REPO / "workflow", root / "workflow", ignore=skip) + shutil.copytree( + REPO / "papers" / "cosmo_val", root / "papers" / "cosmo_val", ignore=skip + ) + (root / "src").symlink_to(REPO / "src") + + catalogue = root / "data" / "toy_shear.fits" + catalogue.parent.mkdir() + catalogue.touch() + (root / "cosmo_val").mkdir() + (root / "cosmo_val" / "cat_config.yaml").write_text( + yaml.safe_dump(_cat_config(catalogue)) + ) + + rundir = root / "papers" / "cosmo_val" + config_path = rundir / "config" / "config.yaml" + config = yaml.safe_load(config_path.read_text()) + config["versions"] = list(VERSIONS) + config["fiducial"]["version"] = VERSIONS[1] + config["fiducial"]["mock_version"] = VERSIONS[0] + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + + env = { + k: v + for k, v in os.environ.items() + if k not in ("SNAKEMAKE_PROFILE", "APPTAINERENV_PYTHONPATH") + } + env.update( + COSMO_VAL=str(root / "out" / "cosmo_val"), + COSMO_INFERENCE=str(root / "out" / "cosmo_inference"), + XDG_CACHE_HOME=str(root / "cache"), + # No local image: launches resolve the registry tag. + SPV_CONTAINER=str(root / "cache" / "absent.sif"), + SPV_SANDBOX=str(root / "cache" / "absent-sandbox"), + TMPDIR=str(tmp_path_factory.getbasetemp()), + PYTHONUNBUFFERED="1", + PYTHONNOUSERSITE="1", + ) + common = _load_module(root / "workflow" / "common.py", "toy_common", env) + + # The processed CosmoCov covariances the cosmo_val rules read, in place. + grids = common.xi_grids(config, config["fiducial"]) + mask = "_masked" if config["covariance"].get("default_masked") else "" + covariances = {} + for version in VERSIONS: + for gaussian, grid in (("ng", grids["reporting"]), ("g", grids["integration"])): + path = covariances[version, gaussian] = Path( + common.covariance_path( + version, + config["fiducial"]["blind"], + gaussian, + grid["min_sep"], + grid["max_sep"], + grid["nbins"], + mask, + ) + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + return Toy( + root=root, + rundir=rundir, + cosmo_val=Path(env["COSMO_VAL"]), + cosmo_inference=Path(env["COSMO_INFERENCE"]), + env=env, + config=config, + common=common, + covariances=covariances, + ) + + +on_candide = pytest.mark.skipif( + not Path("/n17data/cdaley/unions").exists() or shutil.which("apptainer") is None, + reason="needs candide: /n17data and apptainer", +) diff --git a/workflow/tests/data/container_smoke/Snakefile b/workflow/tests/data/container_smoke/Snakefile new file mode 100644 index 00000000..62f4754b --- /dev/null +++ b/workflow/tests/data/container_smoke/Snakefile @@ -0,0 +1,27 @@ +# Standalone workflow exercised by workflow/tests/test_container_smoke.py. +# +# It composes workflow/ as every entry Snakefile does: common's launch-time +# setup, this checkout's src/ on the job's PYTHONPATH, the image a launch +# resolves. The executor, the apptainer deployment and the binds come from the +# driving profile (workflow/profiles/candide). See container_smoke.py for what +# the job checks. + +import os +import sys + +sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), "../../.."))) +import common + +common.inject_checkout_pythonpath(config) + + +container: common.resolve_container(config.get("container")) + + +rule container_smoke: + output: + "results/container_smoke.yaml", + resources: + runtime=5, + script: + "container_smoke.py" diff --git a/src/sp_validation/tests/data/container_smoke/container_smoke.py b/workflow/tests/data/container_smoke/container_smoke.py similarity index 61% rename from src/sp_validation/tests/data/container_smoke/container_smoke.py rename to workflow/tests/data/container_smoke/container_smoke.py index 274cd85c..c25873b3 100644 --- a/src/sp_validation/tests/data/container_smoke/container_smoke.py +++ b/workflow/tests/data/container_smoke/container_smoke.py @@ -2,27 +2,27 @@ Cheap sanity check of the profile-driven container path -- same executor (slurm), same software-deployment-method (apptainer), same apptainer-args -binds, same container image every real rule uses. Four things it proves, each +binds, same container image every real rule uses. What it proves, each written to the output YAML: * the job really ran inside the image (``APPTAINER_CONTAINER``, set by apptainer itself -- without it the rest could all pass on the bare host); - * the editable ``sp_validation`` install resolves on the container's - PYTHONPATH (import provenance: file + version, not just import success); - * the numeric stack works (numpy eigh on a small fixed matrix). - ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; + * which ``sp_validation`` the job imports (file + version, not just import + success); + * which ``snakemake`` package unpickled the injected ``snakemake`` object + (file + version); * which commit of this checkout is running (git rev-parse from inside the container -- proves /home is bound and usable, not just readable). Driven by the co-located Snakefile; the assertions on the output YAML live in -src/sp_validation/tests/test_container_smoke.py (marked ``slow``, cluster only). +workflow/tests/test_container_smoke.py (candide only). """ import os import platform import subprocess +import sys -import numpy as np import yaml # --- the job is actually inside the image --------------------------------- @@ -30,7 +30,7 @@ "apptainer_container": os.environ.get("APPTAINER_CONTAINER", "unset"), } -# --- editable install resolves inside the container ------------------------ +# --- the sp_validation the job imports ------------------------------------- import sp_validation # noqa: E402 sp_validation_info = { @@ -38,24 +38,19 @@ "file": sp_validation.__file__, } -# --- numeric stack + threading ----------------------------------------- -rng = np.random.default_rng(seed=42) -a = rng.standard_normal((8, 8)) -symmetric = a + a.T -eigenvalues = np.linalg.eigh(symmetric)[0] - -numeric_info = { - "numpy_version": np.__version__, - "eigenvalues": [float(v) for v in eigenvalues], - "omp_num_threads": os.environ.get("OMP_NUM_THREADS", "unset"), +# --- the snakemake package the preamble unpickled with --------------------- +# Read from sys.modules: importing it here would rebind the injected global. +snakemake_package = sys.modules["snakemake"] +snakemake_info = { + "version": snakemake_package.__version__, + "file": snakemake_package.__file__, } # --- provenance: what commit is actually running in the container --------- -# src/sp_validation/tests/data/container_smoke/ -> repo root, five levels up. -# (This is the checkout the Snakefile came from, which is what we want to -# report; the editable install may well resolve to a *different* checkout.) +# workflow/tests/data/container_smoke/ -> repo root, four levels up: the +# checkout the Snakefile came from. repo_dir = os.path.abspath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 5)) + os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 4)) ) try: commit = subprocess.run( @@ -79,7 +74,7 @@ { "container": container_info, "sp_validation": sp_validation_info, - "numeric": numeric_info, + "snakemake": snakemake_info, "provenance": provenance, }, f, diff --git a/workflow/tests/pytest.ini b/workflow/tests/pytest.ini new file mode 100644 index 00000000..53d26a02 --- /dev/null +++ b/workflow/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + candide: needs candide (its disks, apptainer and your image); CI deselects these with -m "not candide" diff --git a/workflow/tests/test_container_smoke.py b/workflow/tests/test_container_smoke.py new file mode 100644 index 00000000..b6b6fdeb --- /dev/null +++ b/workflow/tests/test_container_smoke.py @@ -0,0 +1,90 @@ +"""One real SLURM job through the committed candide profile. + +The executor, the apptainer deployment method, the bind mounts and the job +bound come from that profile, launched as the README launches a target; the +test Snakefile composes workflow/ as the entry Snakefiles do, so the job runs +the image that launch resolves (your SIF or sandbox) in the environment that +launch hands its jobs. That contract is what's under test, so it runs only where +``sbatch`` exists: a candide login node. Compute nodes have none, so in a job +step on an allocation it skips. + +The job writes a YAML report (see data/container_smoke/container_smoke.py); the +assertions below check what it reports. +""" + +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest +import snakemake +import yaml +from conftest import REPO, container, on_candide + +SMOKE = Path(__file__).resolve().parent / "data" / "container_smoke" + + +@pytest.mark.candide +@on_candide +@pytest.mark.skipif( + shutil.which("sbatch") is None, reason="submits a SLURM job: run on a login node" +) +def test_container_smoke(): + assert container.resolve_image()[1] != "tag", ( + "no local image; run `spv-container pull`" + ) + + # Not pytest's tmp_path: that lives in the submit host's /tmp, which the + # compute node cannot see, so the job's output would "go missing". The + # workdir must be on a shared filesystem. + workdir = Path(tempfile.mkdtemp(prefix="container_smoke_", dir=Path.home())) + + env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + result = subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "--profile", + str(REPO / "workflow/profiles/candide"), + "-s", + str(SMOKE / "Snakefile"), + "--directory", + str(workdir), + "container_smoke", + ], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=600, + check=False, + ) + assert result.returncode == 0, result.stdout + + report = yaml.safe_load((workdir / "results/container_smoke.yaml").read_text()) + + # The job ran inside the image, not on the bare host. Everything below would + # pass on the host too, so this is the assertion that makes them mean + # something: apptainer sets APPTAINER_CONTAINER in every process it starts. + assert report["container"]["apptainer_container"] != "unset", report["container"] + + # The job imports the launched checkout's sp_validation, not the image's. + module_file = Path(report["sp_validation"]["file"]).resolve() + assert module_file == (REPO / "src/sp_validation/__init__.py").resolve(), ( + module_file + ) + + # The job read the pickle with the Snakemake that wrote it. + assert report["snakemake"]["version"] == snakemake.__version__, report["snakemake"] + + # git worked inside the container, so /home is bound and usable. + assert re.fullmatch(r"[0-9a-f]{40}", report["provenance"]["commit"]), report[ + "provenance" + ] + + shutil.rmtree(workdir) # keep only on failure, for post-mortem diff --git a/workflow/tests/test_dag.py b/workflow/tests/test_dag.py new file mode 100644 index 00000000..4d2ba017 --- /dev/null +++ b/workflow/tests/test_dag.py @@ -0,0 +1,173 @@ +"""DAG properties, checked through the host launcher (see conftest.py).""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from conftest import REPO, VERSIONS, on_candide, parse_jobs + + +def test_assemble_resolves(toy): + """Each terminal file gathers every part and the analytic covariances. + + The ξ± block takes the CosmoCov covariance on the reporting grid, and the + harmonic block is the part on the fiducial harmonic binning with the + NaMaster covariance of that same binning. + """ + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + jobs = [j for j in parse_jobs(result.stdout) if j.rule == "assemble_sacc"] + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + reporting = toy.common.grid_binning(grids["reporting"]) + harmonic = toy.common.pseudo_cl_tag(toy.config) + assert sorted(j.wildcards["version"] for j in jobs) == sorted(VERSIONS) + for job in jobs: + version = job.wildcards["version"] + assert [Path(o).name for o in job.output] == [f"{version}.sacc"] + assert {Path(f).name for f in job.input} == { + f"{version}_xi_{reporting}.sacc", + toy.covariances[version, "ng"].name, + f"pseudo_cl_{version}_{harmonic}.sacc", + f"pseudo_cl_cov_{version}_{harmonic}.fits", + f"{version}_cosebis.sacc", + f"{version}_pure_eb.sacc", + f"rho_tau_{version}_{reporting}.sacc", + }, job.input + + +def test_one_integration_grid(toy): + """COSEBIs and pure-E/B share the integration-grid part and its covariance.""" + result = toy.snakemake("-n", "assemble_sacc_all") + assert result.returncode == 0, result.stdout + jobs = parse_jobs(result.stdout) + grids = toy.common.xi_grids(toy.config, toy.config["fiducial"]) + + tag = toy.common.grid_binning(grids["integration"]) + for version in VERSIONS: + by_rule = { + j.rule: set(j.input) for j in jobs if j.wildcards.get("version") == version + } + part = str(toy.cosmo_val / f"{version}_xi_{tag}.sacc") + covariance = str(toy.covariances[version, "g"]) + assert by_rule["cv_cosebis"] == {part, covariance}, by_rule["cv_cosebis"] + assert {part, covariance} <= by_rule["cv_pure_eb"], by_rule["cv_pure_eb"] + + +@pytest.mark.parametrize("named", [True, False], ids=["named", "unnamed"]) +def test_outputs_stay_in_the_output_roots(toy, named): + """Nothing the suite declares lands outside the configured output roots. + + A launch that names no COSMO_VAL writes into its own checkout's + cosmo_val/output. + """ + env = toy.env if named else {k: v for k, v in toy.env.items() if k != "COSMO_VAL"} + cosmo_val = toy.cosmo_val if named else toy.root / "cosmo_val" / "output" + result = toy.snakemake("-n", "all", env=env) + assert result.returncode == 0, result.stdout + roots = [ + r.resolve() for r in (cosmo_val, toy.cosmo_inference, toy.rundir / "results") + ] + outputs = [Path(o) for j in parse_jobs(result.stdout) for o in j.output] + assert outputs, result.stdout + strays = [ + o + for o in outputs + if not any((toy.rundir / o).resolve().is_relative_to(r) for r in roots) + ] + assert not strays, strays + + +def _apptainer_stub(tmp_path): + """A PATH entry that answers where apptainer is not installed. + + The candide profile deploys with apptainer, whose version Snakemake reads + even when it runs no job. + """ + apptainer = tmp_path / "bin" / "apptainer" + apptainer.parent.mkdir() + apptainer.write_text("#!/bin/sh\necho apptainer version 1.3.4\n") + apptainer.chmod(0o755) + return apptainer.parent + + +def test_a_job_needs_no_launch_cache(toy, tmp_path): + """A job starts on a node where the launch's XDG cache cannot exist. + + Jobs inherit the launching shell's environment, whose XDG_CACHE_HOME may be + node-local. Under the candide profile, a Snakemake that cannot create that + cache still starts, and a job's environment carries no XDG_CACHE_HOME for + the Snakemake its job step starts. + """ + blocker = tmp_path / "a-file" + blocker.touch() + candide = toy.root / "workflow" / "profiles" / "candide" + result = toy.snakemake( + "-n", + "--profile", + str(candide), + "assemble_sacc_all", + env=toy.env + | { + "XDG_CACHE_HOME": str(blocker / "cache"), + "PATH": f"{_apptainer_stub(tmp_path)}{os.pathsep}{toy.env['PATH']}", + }, + ) + assert result.returncode == 0, result.stdout + + snakefile = tmp_path / "Snakefile" + snakefile.write_text( + f"import sys\nsys.path.insert(0, {str(toy.root / 'workflow')!r})\n" + "import common\n\n" + 'rule job:\n output: "env.txt"\n' + ' shell: "printenv XDG_CACHE_HOME > {output} || true"\n' + ) + result = toy.snakemake( + "-s", + str(snakefile), + "--directory", + str(tmp_path), + env=toy.env | {"XDG_CACHE_HOME": str(tmp_path / "launch-cache")}, + ) + assert result.returncode == 0, result.stdout + assert (tmp_path / "env.txt").read_text() == "" + + +def _real_dry_run(paper, targets): + env = {k: v for k, v in os.environ.items() if k != "SNAKEMAKE_PROFILE"} + env.update(PYTHONUNBUFFERED="1", PYTHONNOUSERSITE="1") + return subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "-n", + "--profile", + str(REPO / "workflow" / "profiles" / "candide"), + *targets, + ], + cwd=REPO / "papers" / paper, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=600, + check=False, + ) + + +@pytest.mark.candide +@on_candide +@pytest.mark.parametrize( + "paper, targets", + [ + ("cosmo_val", ["assemble_sacc_all"]), + ("bmodes", ["paper"]), + ], + ids=["cosmo_val", "bmodes"], +) +def test_papers_resolve_on_candide(paper, targets): + """The real paper DAGs resolve against the real catalogues and your image.""" + result = _real_dry_run(paper, targets) + assert result.returncode == 0, result.stdout From a8a08bc436f38b246eabd54f9a6e712768ac0aaf Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Mon, 28 Sep 2026 23:46:08 +0200 Subject: [PATCH 40/83] Scrub the legacy n(z) A/B/C blind: an n(z) is a catalogue entry's (#359) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests: pin pure-E/B on exact-binning and committed ξ±; drop dead catalogue paths The synthetic pure-E/B pins passed in CI and failed on candide because TreeCorr's default bin_slop/angle_slop make ξ± follow the tree's top-level split, which varies with the jackknife patches and, through min_top, with the thread count TreeCorr takes from cpu_count(). Fixed patch centres alone leave a 4-vs-48-thread spread (reporting ξ− up to 16%); exact binning removes it (1e-12). The test measures with bin_slop = angle_slop = 0, and test_b_modes pins pure_eb_from_xi on the same ξ±, committed as tests/data/pure_eb_xi_fixture.npz. The configured-path guard skips cat_config's paths.output and directory-less calibration params.input_path values. The two LFmask entries (data gone) and the six unread covmat_file keys leave cat_config.yaml, and the slow duplicate test_catalog_paths_exist goes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * cosmo_val: pin TreeCorr's min_top so ξ± does not depend on the machine TreeCorr derives min_top, the depth of its root cells, from its thread count (max(3, ceil(log2 n)), with n from cpu_count() when unset). The root cells set which pairs bin_slop approximates, so at default slop calculate_2pcf's ξ± depended on the node: on the synthetic catalogue, 48 threads move ξ± by 0.038σ against 4. The shared treecorr_config pins min_top = 6, which is what TreeCorr derives on candide's 48- and 64-CPU nodes; the same config reaches the aperture-mass, leakage and ρ/τ correlations. test_calculate_2pcf_does_not_depend_on_thread_count measures at production binning on 4 and 48 threads (fresh Catalog each, shared patch centres) and requires agreement below 1e-6σ; it is red without the pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * b_modes: one pure-E/B kernel call; tests pin each pure-E/B fact once calculate_pure_eb_correlation (modes and jackknife) and pure_eb_covariance_mc call cosmo_numba through pure_eb_from_xi, so the transform test_b_modes pins is the one every pure-E/B product runs through. The committed ξ± fixture is now a conftest fixture (pure_eb_xi) with two users. The synthetic pure-E/B test asserts that the ξ± and edges it measures equal the fixture (agreement 2e-13), and that its modes are pure_eb_from_xi of them; this replaces its copy of the mode pins, and np.savez of what it measures regenerates the fixture. test_b_modes pins pure_eb_from_xi on the fixture, which fails loudly if cosmo_numba does not import, with no skip. A failure now names what moved: the measured ξ±, the wiring into the kernel, or the transform. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * b_modes: one pure-E/B transform per jackknife realisation The jackknife covariance function indexed pure_EB(x) once per key, so every realisation ran cosmo_numba's transform six times (55 transforms at npatch=8 where 10 suffice; values unchanged). _eb_vector concatenates the modes in _EB_KEYS order for both the jackknife and the MC covariance. The synthetic pure-E/B test counts transforms around calculate_pure_eb and is red on the per-key closure (55 > npatch + 2). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * tests: glue-section header stops enumerating which tests compare values Two tests in the section compare values (pure-E/B against committed ξ±, ξ± across TreeCorr thread counts); the header defers to each test's docstring instead of listing them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: host/image parity check, catalogue config from the checkout, one output root, one integration grid - Launch-time parity: container.image_runtime reads the image's Python minor and Snakemake version (SIF via apptainer, sandbox off disk, tags skipped); common.check_host_parity stops a launch whose host Snakemake differs, with the reinstall command. configure() and image_sims.smk call it; the README install line pins --python 3.12. - The catalogue config is the launched checkout's cosmo_val/cat_config.yaml, loaded by configure() into CATALOG_CONFIG; the paper Snakefiles no longer merge it into `config`, and covariance.smk / ecut.smk read CATALOG_CONFIG. - One output root: cv_init_params passes output_dir=COSMO_VAL, cv_runner no longer chdirs into the live checkout, and CosmologyValidation drops its COSMO_VAL environment fallback. - One integration grid (R12): the cosebis grid is gone; cv_cosebis reads the integration part and the CosmoCov g covariance on that grid (the one pure-E/B uses). An npatch=1 grid defaults to the diagonal covariance, and a binning outside the named grids takes its covariance from its own patches. - The candide profile sets jobs: 100. - workflow/tests: host-launcher DAG tests on a toy checkout (P1-P3) and the real papers on candide (P4, P5 = the container smoke test, moved here); CI runs them in a workflow-dag job. test_bmodes_workflow_dry_run.py is replaced by P4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: the image lives under ~/.cache; launch guards read the real image - container.CACHE_DIR is ~/.cache/sp_validation whatever XDG_CACHE_HOME says. A job runs the image from the path the launching host resolved, and on candide XDG_CACHE_HOME is node-local /scratch: from such a shell the launch fell back to the registry tag and skipped the parity check. SPV_CONTAINER / SPV_SANDBOX remain the overrides. - test_launch_reads_the_image_under_home: a mismatched image under a fake ~/.cache stops the launch while XDG_CACHE_HOME points elsewhere. - test_papers_resolve_on_candide asserts unconditionally that the launch read a local image (no "parity unchecked"). - test_image_sims_checks_parity_at_launch: the standalone image-sims Snakefile stops on a mismatched image. - test_assemble_resolves pins each terminal file's inputs: the reporting ξ± part with its CosmoCov ng covariance, the fiducial-binning pseudo-Cl part with its NaMaster covariance, COSEBIs, pure-E/B and ρ/τ. - The container smoke test launches as the README does (no --jobs), so the candide profile's job bound is under test. - Test docstrings drop the design-table row IDs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: a ξ± part's covariance follows its patches, in one place A grid is a binning; rule xi takes cov=patch_cov(npatch) directly, so grid_cov, _named_grid and the grids' cov key go. cv_init_params loses its unused version_list, and the cosmo_val rules pass one CV_INIT. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/README: scope the output-root sentence to papers/cosmo_val Name where the other rules write: masks under the run directory's output/masks/, papers/bmodes' figures and macros under its docs/, image sims under grids_base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: SLURM jobs start whatever the launch's XDG_CACHE_HOME; P4/P5 test what a launch does A job's Snakemake inherits the launching shell's environment (--export=ALL), and a login shell may point XDG_CACHE_HOME at node-local storage: every job then fails creating its source cache. Two halves close it, each needed (shown on n33 by replaying the captured sbatch job command with srun stubbed): - the candide profile leaves source-cache out of shared-fs-usage, so a job neither reuses the launch's cache path nor creates its own under XDG; - common.py drops XDG_CACHE_HOME from what jobs inherit, because the slurm-jobstep executor forces a shared source cache on the Snakemake it starts for each job step. test_a_job_needs_no_launch_cache covers both (each mutation turns it red). P4 (test_papers_resolve_on_candide) now passes from a shell with node-local XDG_CACHE_HOME under srun, and its bmodes case also resolves an e-cut catalogue, so both CATALOG_CONFIG readers in ecut.smk are exercised. P5 hands the smoke Snakefile the image a launch resolves (no registry pull). The host launcher and CI carry snakemake-executor-plugin-slurm, as the README install line does, so the candide profile is parsed in CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * papers/bmodes: the sweep drivers read the image from ~/.cache, as container.py does container_env.sh still followed XDG_CACHE_HOME, so from a login shell that points it at node-local storage the sweep drivers ran a missing image while spv-container and the workflow found the real one. test_sweep_drivers_run_the_resolved_image pins the shell copy of the resolution (cache and sandbox precedence) to container.resolve_image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * run_2pcf: a ξ± part carries the covariance its measurement estimated Which covariance a part carries had two homes that disagreed: rule xi passed cov=patch_cov(npatch) ("diagonal" at npatch=1), while run_2pcf's own default, reached by the CLI and papers/bmodes' run_xi_sweep, was "none". The same file name then held variances or not depending on who wrote it. run_2pcf now carries what TreeCorr estimated (gg.var_method, which calculate_2pcf sets from npatch): the jackknife covariance with patches, the shot-noise diagonal without. The cov argument, --cov, the jackknife guard, rule xi's cov param and common.patch_cov (with its test) go. Rule xi hands its wildcards to grid_of directly (xi_binning_of goes). test_xi_part_carries_the_covariance_the_measurement_estimated runs run_2pcf on the synthetic catalogue at npatch 1 and 4; the previous run_2pcf fails both cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: the launch-cache test answers the profile's apptainer check with a stub The candide profile deploys with apptainer, and Snakemake asks for the apptainer binary and its version even in a dry-run, so test_a_job_needs_no_launch_cache failed on GitHub's runners, which have no apptainer. A stub apptainer on the test's PATH answers that check; the dry-run reads nothing else from it. In a CI emulation (a clean checkout, a fresh HOME, no apptainer or SLURM on PATH, the workflow-dag command) the suite goes from 1 failed, 11 passed to 12 passed. Both mutations still turn the test red there: source-cache added back to the profile's shared-fs-usage (NotADirectoryError under the blocked XDG_CACHE_HOME) and the XDG_CACHE_HOME pop removed from common.py (the job sees the launch's cache). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: the smoke Snakefile composes workflow/ as the entry Snakefiles do P5's Snakefile never imported common, so its launch did not drop XDG_CACHE_HOME: from a login shell pointing it at node-local /scratch, the job's inner Snakemake died creating its source cache (PermissionError), which says nothing about what a real launch does. The Snakefile now imports common, resolves its image with common.resolve_container and checks host/image parity, as the entry Snakefiles do. P5 launches exactly as the README does, with no --config container=, and the literal default image and the test that kept it in step with CONTAINER_URI go. Checked on n33 with sbatch stubbed to record what it would submit, then the recorded job replayed with srun stubbed, from a shell with XDG_CACHE_HOME set to /scratch/cdaley/tmp/xdg: at the parent commit the job environment carries XDG_CACHE_HOME and the replay fails with PermissionError; with this commit it carries none, the job runs in the SIF, and P5's assertions pass on its report. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: resolve_container checks the image it returns The host/image parity check had a home beside the resolver, so configure() resolved the image once only to check it, and every Snakefile that picks its own image (image_sims.smk, the smoke Snakefile) had to remember a second line. resolve_container now runs check_host_parity on the image it returns: the image checked is the image that runs, and the extra call sites go. check_host_parity stays cached, since composed Snakefiles evaluate container: more than once. Removing the check from resolve_container turns 5 host tests red (both parity cases, the unreadable-image line, the image under ~/.cache, image sims). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: P5 checks that the job imports the launched checkout's src The smoke Snakefile imported common but never put the checkout's src/ on the job's PYTHONPATH, so its job imported the image's baked sp_validation, and P5's assertion (any editable src/ layout) passed on it. The smoke Snakefile now calls inject_checkout_pythonpath as configure() does for the entry Snakefiles, and P5 asserts the job's sp_validation is this checkout's src/sp_validation/__init__.py. Run on n08 through the default profile (apptainer, no SLURM): the job reports /src/sp_validation/__init__.py; with the injection removed it reports /sp_validation/src/sp_validation/__init__.py, which the new assertion rejects and the old one accepted. P5 itself needs a SLURM submit host and has not been run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow: COSMO_VAL defaults to the launched checkout's own output tree A launch from any checkout that named no COSMO_VAL read that checkout's catalogue config and code but wrote every cosmo_val product into the production tree, silently. COSMO_VAL now defaults to the launched checkout's (gitignored) cosmo_val/output, so production is written only from the production checkout or when a launch names it; COSMO_INFERENCE keeps its shared default. README says so. P2 gains an unnamed case: with COSMO_VAL unset, every declared output lies under the toy checkout's cosmo_val/output, the inference root or results/. Restoring the production default turns it red (outputs under /n17data/cdaley/unions/code/sp_validation/cosmo_val/output). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * calculate_2pcf: the ξ± text dump carries columns only, so it reads back With patches, calculate_2pcf wrote its .txt with write_cov=True and no per-patch results. TreeCorr 5.1.4 writes num_rows only alongside patch results, so its reader ran on into the cov block and raised ("got 12 columns instead of 11"). The two ξ± figure rules re-enter calculate_2pcf on the reporting grid (npatch=100) and hit exactly that read. The dump now carries the columns only; the covariance matrix lives in the SACC part, and the figure readers use the columns. test_a_patched_xi_dump_reads_back measures at npatch=4, then re-enters calculate_2pcf from a fresh CosmologyValidation and compares the columns; under write_cov=True it fails with the ValueError above. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * workflow/tests: the candide profile's job bound is checked on any host A real launch through the committed candide profile, of an up-to-date target, submits nothing, so it runs in CI and on an allocation; the same launch through the profile without `jobs` is refused. test_container_smoke submits a real job and needs sbatch, which only a login node has; it is documented as the login-node check it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013EGDJZeA8v26eKGAHQqh5j * profiles: wait 60 s for a job's outputs to appear P5 (one real SLURM job through the candide profile, from a login node) passed, but only on its retry: the job finished, its output took more than 5 s to show on the login node's /home, and Snakemake re-ran it. On a multi-hour job that retry costs hours, and a second miss fails the run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * rho_tau_stats: 6 h wall clock; the jackknife ρ/τ outruns the 60 min default Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * Scrub the n(z) A/B/C blind: an n(z) is a catalogue entry's Choosing an n(z) is choosing a catalogue entry: each entry's shear.redshift_path is its n(z), read by CosmologyValidation.get_redshift and by the workflow's redshift_path(version). CosmologyValidation loses its blind override, and the pseudo-Cl, covariance, inference and papers/bmodes filenames lose their blind token and wildcard. - common: catalogue_entry()/redshift_path() replace build_redshift_path; covariance_{base,dir,path} and pseudo_cl_tag drop blind; the version constraint admits the _A/_B/_C entries. - covariance.smk: get_cat_params and the star-halo mask resolve through the catalogue entry. - papers/bmodes: bb_covariance_blind_independence and the talk n(z) plot compare fiducial.nz_realisations, the SP_v1.4.6.3_{A,B,C}_leak_corr entries; pure-E/B, COSEBIs PTE and pseudo-Cl paths are per version. - cat_config: SP_v1.4.6.3_{A,B,C} match SP_v1.4.6.3 but for their n(z); SP_v1.4.8 and SP_v1.4.11.3(_ecut07) name the n(z) their covariances used. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * cat_config: the leakage n(z) is a path template, not an A/B/C blind key nz.dndz.path names the file with {pipeline}; the same file as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * cat_config: SP_v1.4.11.2 reads the n(z) its covariance used Like SP_v1.4.11.3, its covariance was built from nz_SP_v1.4.6_A.txt. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * workflow: no snakemake in the image, so any host Snakemake works A script: job unpickles the host's snakemake object with whichever snakemake it imports first, and the image's site-packages precede the host's appended sys.path. The image carried its own (the workflow extra, and the base image's jupyter extra), so the host had to match its version exactly. The Dockerfile now uninstalls every snakemake* package after the sync and the workflow extra drops snakemake; the job then reads the pickle with the package that wrote it. The launch check keeps only the Python minor (check_host_python): the host's snakemake and its compiled dependencies load into the image's interpreter. The README install line, CI's DAG job and the test harness no longer pin a Snakemake version. The container smoke job now reports which snakemake unpickled its object and asserts it is the host's version. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * profiles: every key needed and non-default, each with its reason Drops the candide LD_LIBRARY_PATH to /softs/openmpi: /softs is not bound, so the path does not exist inside the container, and no containerized rule uses MPI. Drops the default profile's --bind /home (apptainer mounts $HOME already). States the real reasons for the rest: rerun-triggers leaves out software-env because it hashes the per-person image path; shared-fs-usage leaves out source-cache because jobs would be handed the launch's node-local cache path; slurm_account because the executor's guess fails on candide; retries and kept logs for fan-outs and their printed output. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * workflow: jobs read the repo's matplotlibrc, never the launching user's Apptainer binds $HOME, so a job read the launcher's own matplotlibrc, and a LaTeX preamble there the image cannot typeset stopped every figure rule. common.py points each job's MATPLOTLIBRC (through APPTAINERENV_, past --cleanenv) at an empty workflow/matplotlibrc. The container smoke job reports the matplotlibrc it would read and the test asserts it is the repo's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * CONTRACTS: the host-side workflow imports only stdlib and snakemake workflow.common runs in the host Snakemake with no sp_validation installed, and loads container.py by path; container.py imports only the standard library, since it also runs as the spv-container CLI before any image exists. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * profiles/default: bind /home; Snakemake's --home keeps apptainer from mounting it A job's home is its working directory, so a checkout or output tree under the launching user's home was invisible to the job: the toy run's jobs imported the image's sp_validation instead of the checkout's src/. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TJSfyQjoQXjPKsEfkGhZLj * workflow: jobs read the user's matplotlibrc; no snakemake-uninstall step The image's TeX now carries sfmath, so a usetex preamble typesets inside jobs and the repo matplotlibrc override has nothing left to guard. The shapepipe:develop base ships no snakemake, so the uninstall step is gone; pyproject says why the workflow extra must not re-add it. Co-Authored-By: Claude Opus 5.5 * workflow: resolve the image without a host/image Python check; trim profile and README The launch resolves its image and runs it; the README states that the host Snakemake runs on the image's Python (3.12). The CONTRACTS notes go (the workflow-dag CI job is what enforces host-importability). The candide profile excludes n17 and n36 only, and the README's output-roots paragraph and the profile's shared-fs-usage comment say what they need to in fewer lines. The DAG tests drop the Python-check tests and the profile job-bound test, and test_one_integration_grid checks only the shared COSEBIs/pure-E/B inputs. Co-Authored-By: Claude Opus 5.5 * tests: drop the smoke test's numeric check and perf-pinning asserts The container smoke job reports and checks placement, imports, Snakemake version and provenance; the host-side suite no longer needs numpy. test_xi_grids loses its grid-count test, and the pure-E/B cosmo_val test keeps its value checks without counting kernel calls. Co-Authored-By: Claude Opus 5.5 * generate_paper_macros: read the single pure E/B joint PTE, drop per-blind macros pure_eb_data_vector writes one fiducial `pte_joint`, so \ebfiducialPte and \ebfullPte read that key. The covariance_blind_consistency and per-blind PTE-spread macros go: nothing produces their inputs. Co-Authored-By: Claude Opus 5.5 * paper macros: drop the unused COSEBIS macros and the inputs no script reads generate_paper_macros no longer emits \cosebisfiducialPte, \cosebisfullPte or \cosebisthetaMin/Max: they read keys cosebis_version_comparison evidence does not carry and no paper uses them (the COSEBIS PTEs come from \configPteCosebis[Full]). The macro rules drop the cosebis and BB realisation evidence inputs accordingly. Co-Authored-By: Claude Opus 5.5 * bb_covariance_nz_independence: realisations from fiducial.nz_realisations The BB-covariance n(z)-independence check (renamed from bb_covariance_blind_independence: rule, script, tapestry outputs) takes its realisation labels from config fiducial.nz_realisations, the first as the reference and every other compared against it; figure markers, evidence keys (