From 0296a918380249f8b77a8d58f357c756267dfdf2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 6 Oct 2025 19:39:32 +0200 Subject: [PATCH 01/29] use consistent config for 1.4.5/6/7/8 and reorganize entries --- notebooks/cosmo_val/cat_config.yaml | 197 ++++++++++++++++++++-------- 1 file changed, 141 insertions(+), 56 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 13991a4..7893d92 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -7,6 +7,49 @@ nz: path: dndz blind: A +SP_test: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/ + ls: dashed + colour: black + marker: d + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: 2420.2651014497287 #deg^2 + n_e: 7.040818382014773 #arcmin-2 + n_psf: 0.752316232272063 #arcmin-2 + sigma_e: 0.30961528707207325 + shear: + path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1 + e2_col: e2 + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_iv + R: 1.0 + redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + star: + path: unions_shapepipe_star_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: unions_shapepipe_psf_2024_v1.4.1.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + SP_v1.0: pipeline: SP subdir: CFIS/v1.0/ShapePipe @@ -722,36 +765,36 @@ SP_v1.4.5: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.6: +SP_v1.4.5_glass_mock: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: CFIS/v1.0/ShapePipe ls: dashed - colour: green + colour: crimson marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 + A: 2782 #deg^2 + n_e: 7.18 #arcmin-2 n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 + sigma_e: 0.4370966656902571 shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: unions_v1.4.5_glass_mock.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.1.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.1.fits hdu: 1 ra_col: RA dec_col: Dec @@ -764,27 +807,27 @@ SP_v1.4.6: PSF_flag: "FLAG_PSF_HSM" star_flag: "FLAG_STAR_HSM" square_size: True - -SP_v1.4.5.A: + +SP_v1.4.5_leak_corr: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 ls: dashed - colour: red + colour: teal marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 + A: 2782 #deg^2 + n_e: 7.18 #arcmin-2 n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 + sigma_e: 0.4370966656902571 shear: - path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: g1 - e2_col: g2 + e1_col: e1_leak_corrected + e2_col: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w + w_col: w_des R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: @@ -794,7 +837,7 @@ SP_v1.4.5.A: e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -937,37 +980,36 @@ SP_v1.4.5_intermediate: star_flag: "FLAG_STAR_HSM" square_size: True - -SP_v1.4.5_glass_mock: +SP_v1.4.5.A: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: CFIS/v1.0/ShapePipe/ ls: dashed - colour: crimson + colour: red marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 + A: 2420.2651014497287 #deg^2 + n_e: 7.040818382014773 #arcmin-2 n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 + sigma_e: 0.30961528707207325 shear: - path: unions_v1.4.5_glass_mock.fits + path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 + e1_col: g1 + e2_col: g2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w_col: w + w: w R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.1.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -980,37 +1022,37 @@ SP_v1.4.5_glass_mock: PSF_flag: "FLAG_PSF_HSM" star_flag: "FLAG_STAR_HSM" square_size: True - -SP_test: + +SP_v1.4.6: pipeline: SP subdir: CFIS/v1.0/ShapePipe/ ls: dashed - colour: black + colour: green marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 + A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD shear: - path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w_col: w_iv + w_col: w_des R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.1.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1024,20 +1066,20 @@ SP_test: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.5_leak_corr: +SP_v1.4.6_leak_corr: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 + subdir: CFIS/v1.0/ShapePipe/ ls: dashed - colour: teal + colour: green marker: d getdist_colour: 0.0, 0.5, 1.0 cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 + A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1_leak_corrected e2_col: e2_leak_corrected @@ -1047,13 +1089,56 @@ SP_v1.4.5_leak_corr: R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + +SP_v1.4.7_leak_corr: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/ + ls: dashed + colour: darkorchid + marker: d + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: null # TBD + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD + shear: + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1_leak_corrected + e2_col: e2_leak_corrected + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_des + R: 1.0 + redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + star: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec From 8f9aab0fcf41b30e342fd10a808b56a5b0b736de Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 6 Oct 2025 19:40:38 +0200 Subject: [PATCH 02/29] add 1.4.8 --- notebooks/cosmo_val/cat_config.yaml | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 7893d92..88c3068 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1152,6 +1152,49 @@ SP_v1.4.7_leak_corr: star_flag: "FLAG_STAR_HSM" square_size: True +SP_v1.4.8_leak_corr: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/ + ls: dashed + colour: darkorchid + marker: d + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: null # TBD + n_e: null # TBD + n_psf: null # TBD + sigma_e: null # TBD + shear: + path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1_leak_corrected + e2_col: e2_leak_corrected + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_des + R: 1.0 + redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + star: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + SP_v1.5.4: pipeline: SP subdir: ../../../UNIONS/WL/v1.5.x From 1b3f5397a4a322fc5844ea55e78ecbcbfc71ef1b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Oct 2025 00:12:51 +0200 Subject: [PATCH 03/29] refactor: standardise catalog paths and add guardrails --- docs/PR_DRAFT.md | 38 ++ notebooks/cosmo_val/cat_config.yaml | 366 +++++++----------- notebooks/cosmo_val/run_cosmo_val.py | 8 +- src/sp_validation/cosmo_val.py | 80 +++- src/sp_validation/tests/test_catalog_paths.py | 97 +++++ src/sp_validation/tests/test_cosmo_val.py | 127 ++++++ 6 files changed, 474 insertions(+), 242 deletions(-) create mode 100644 docs/PR_DRAFT.md create mode 100644 src/sp_validation/tests/test_catalog_paths.py create mode 100644 src/sp_validation/tests/test_cosmo_val.py diff --git a/docs/PR_DRAFT.md b/docs/PR_DRAFT.md new file mode 100644 index 0000000..38b1bfe --- /dev/null +++ b/docs/PR_DRAFT.md @@ -0,0 +1,38 @@ +# Draft PR: Catalog Path Cleanup & Guardrails + +## Summary + +- Normalised every `subdir` entry in `notebooks/cosmo_val/cat_config.yaml` so the loader relies on explicit `/n17data/...` locations rather than the old `data_dir` fallback. Shared resources (redshift distributions, common star/psf FITS) now point at their canonical mounts. +- Updated the Pseudo-`C_\ell` code path in `src/sp_validation/cosmo_val.py` to read redshift files directly from the catalog configuration, removing the `data_base_dir` assumption. +- Added `src/sp_validation/tests/test_catalog_paths.py`, a lightweight validator that resolves each shear/star/psf reference and records a small allow-list for catalogues whose source files are currently unavailable. +- Kept the additive-bias regression test focused on the catalogues that still have data on disk; the missing releases stay excluded so CI remains green. + +## Outstanding Data Gaps + +All of the catalogues below are missing their **shear** FITS on the shared mounts (star/psf often missing as well): + +1. `SP_v1.0`, `SP_v1.1`, and `SP_matched_MP_v1.0` +2. The `SP_v1.4` family: `SP_v1.4`, `SP_v1.4_conv`, `SP_v1.4_noalpha` +3. Every `SP_v1.4-P1+3*` derivative (same missing 2022-era inputs as item 2) + +These entries remain in `cat_config.yaml` for provenance, are listed in the validator allow-list, and are intentionally omitted from the additive-bias parameter set. The follow-up PR should either restore their data to `/n17data/...`, retarget them at new releases (e.g. the 2024 v1.4.1 products), or mark them as deprecated. + +## Proposed Next Steps + +1. **Decide The Fate Of The Legacy Catalogues** + - Recover the missing FITS/parquet files, or formally deprecate the versions above (update docs/tests accordingly). + +2. **Re-home The v1.4 Variants (if data exist)** + - Point each affected block at the correct `/n17data/UNIONS/WL/v1.4.x//` directory. + - Prefer catalogue-local star/psf fits when the directory is more than a symlink; fall back to the shared root otherwise. + - Remove the entries from `EXPECTED_MISSING` and re-enable them in regression tests once their files resolve. + +3. **Document The Conventions** + - Add a short developer note describing the absolute-path policy, preferred star/psf strategy, and how to run the new path validator before pushing catalog edits. + - Optionally list the currently offline datasets so collaborators know what needs recovering. + +## Validation Checklist + +- `app python -m pytest src/sp_validation/tests/test_catalog_paths.py` +- `app python -m pytest src/sp_validation/tests/test_cosmo_val.py -k additive_bias` +- Manual `ls` spot-check of any retargeted catalogue directories to ensure we pick up the intended local star/psf files rather than parent symlinks. diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 88c3068..493b5db 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -2,14 +2,16 @@ paths: output: ./output nz: - subdir: CFIS/v1.0/nz + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + dndz: path: dndz blind: A SP_test: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: dashed colour: black marker: d @@ -28,7 +30,7 @@ SP_test: e2_PSF_col: e2_PSF w_col: w_iv R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.1.fits ra_col: RA @@ -52,7 +54,8 @@ SP_test: SP_v1.0: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: blue marker: o @@ -87,7 +90,8 @@ SP_v1.0: SP_v1.1: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: dashed colour: b marker: s @@ -120,7 +124,8 @@ SP_v1.1: SP_v1.3: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: green marker: p @@ -158,7 +163,8 @@ SP_v1.3: SP_v1.4: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: brown marker: d @@ -205,7 +211,8 @@ SP_v1.4: SP_v1.4_conv: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: black marker: d @@ -252,7 +259,8 @@ SP_v1.4_conv: SP_v1.4_noalpha: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: solid colour: brown marker: d @@ -300,7 +308,8 @@ SP_v1.4_noalpha: SP_v1.4-P3: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 + ls: dashed colour: brown marker: d @@ -346,7 +355,8 @@ SP_v1.4-P3: SP_v1.4-P1+3: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: brown marker: d @@ -390,7 +400,8 @@ SP_v1.4-P1+3: SP_v1.4-P1+3_wcs: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: brown marker: d @@ -432,7 +443,8 @@ SP_v1.4-P1+3_wcs: SP_v1.4-P1+3_no_alpha: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: black marker: d @@ -474,7 +486,8 @@ SP_v1.4-P1+3_no_alpha: SP_v1.4-P1+3_li_2024: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + ls: dashed colour: gold marker: d @@ -516,7 +529,8 @@ SP_v1.4-P1+3_li_2024: SP_v1.4-P1+3+4: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 + ls: dashed colour: brown marker: d @@ -556,7 +570,8 @@ SP_v1.4-P1+3+4: SP_v1.4-P1+3+4_wcs: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 + ls: dashed colour: black marker: d @@ -598,7 +613,8 @@ SP_v1.4-P1+3+4_wcs: SP_v1.4-P1+3+4_no_alpha: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 + ls: dashed colour: black marker: d @@ -640,7 +656,8 @@ SP_v1.4-P1+3+4_no_alpha: SP_v1.4.2: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + ls: dashed colour: black marker: d @@ -682,7 +699,8 @@ SP_v1.4.2: SP_v1.4.1_noleakage: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + ls: dashed colour: black marker: d @@ -702,13 +720,13 @@ SP_v1.4.1_noleakage: w_col: w_iv R: 1.0 star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.1.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.1.fits hdu: 1 ra_col: RA dec_col: Dec @@ -724,7 +742,8 @@ SP_v1.4.1_noleakage: SP_v1.4.5: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: dodgerblue marker: d @@ -735,15 +754,17 @@ SP_v1.4.5: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.4370966656902571 shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -767,7 +788,8 @@ SP_v1.4.5: SP_v1.4.5_glass_mock: pipeline: SP - subdir: CFIS/v1.0/ShapePipe + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: crimson marker: d @@ -778,7 +800,7 @@ SP_v1.4.5_glass_mock: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.4370966656902571 shear: - path: unions_v1.4.5_glass_mock.fits + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 @@ -786,50 +808,7 @@ SP_v1.4.5_glass_mock: e2_PSF_col: e2_PSF w_col: w R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_leak_corr: - pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 - ls: dashed - colour: teal - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -853,7 +832,8 @@ SP_v1.4.5_leak_corr: SP_v1.4.5_bright: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/murray/unions_cats + ls: dashed colour: mediumblue marker: d @@ -864,23 +844,23 @@ SP_v1.4.5_bright: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -896,7 +876,8 @@ SP_v1.4.5_bright: SP_v1.4.5_faint: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/murray/unions_cats + ls: dashed colour: forestgreen marker: d @@ -907,23 +888,23 @@ SP_v1.4.5_faint: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -939,7 +920,8 @@ SP_v1.4.5_faint: SP_v1.4.5_intermediate: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/murray/unions_cats + ls: dashed colour: orange marker: d @@ -950,23 +932,23 @@ SP_v1.4.5_intermediate: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_des + w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -982,7 +964,8 @@ SP_v1.4.5_intermediate: SP_v1.4.5.A: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 + ls: dashed colour: red marker: d @@ -993,23 +976,23 @@ SP_v1.4.5.A: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits + path: shapepipe_SPv1.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: g1 e2_col: g2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w + w_col: w R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: unions_shapepipe_star_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1025,7 +1008,8 @@ SP_v1.4.5.A: SP_v1.4.6: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: green marker: d @@ -1036,15 +1020,17 @@ SP_v1.4.6: n_psf: null # TBD sigma_e: null # TBD shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -1052,7 +1038,7 @@ SP_v1.4.6: e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1066,52 +1052,10 @@ SP_v1.4.6: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.6_leak_corr: +SP_v1.4.7: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x -SP_v1.4.7_leak_corr: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ ls: dashed colour: darkorchid marker: d @@ -1122,23 +1066,25 @@ SP_v1.4.7_leak_corr: n_psf: null # TBD sigma_e: null # TBD shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected + e1_col: e1 + e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1152,9 +1098,10 @@ SP_v1.4.7_leak_corr: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.4.8_leak_corr: +SP_v1.4.8: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: /n17data/UNIONS/WL/v1.4.x + ls: dashed colour: darkorchid marker: d @@ -1165,23 +1112,25 @@ SP_v1.4.8_leak_corr: n_psf: null # TBD sigma_e: null # TBD shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected + e1_col: e1 + e2_col: e2 + e1_col_corrected: e1_leak_corrected + e2_col_corrected: e2_leak_corrected e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF w_col: w_des R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits + path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 ra_col: RA dec_col: Dec @@ -1197,7 +1146,8 @@ SP_v1.4.8_leak_corr: SP_v1.5.4: pipeline: SP - subdir: ../../../UNIONS/WL/v1.5.x + subdir: /n17data/UNIONS/WL/v1.5.x + ls: dashed colour: green marker: d @@ -1237,51 +1187,10 @@ SP_v1.5.4: star_flag: "FLAG_STAR_HSM" square_size: True -SP_v1.5.4_leak_corr: - pipeline: SP - subdir: ../../../UNIONS/WL/v1.5.x - ls: dashed - colour: darkgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.5.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.5.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - LF_v1.0: pipeline: LF - subdir: CFIS/v1.0/Lensfit + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit + ls: solid colour: r marker: "8" @@ -1301,7 +1210,8 @@ LF_v1.0: LF_v2.0: pipeline: LF - subdir: CFIS/v2.0/Lensfit + subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit + ls: dashed colour: r marker: p @@ -1319,7 +1229,8 @@ LF_v2.0: SP_matched_LF_v1.0: pipeline: SP - subdir: CFIS/v1.0/matched_LF_SP + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + ls: dotted colour: magenta marker: v @@ -1352,7 +1263,8 @@ SP_matched_LF_v1.0: LF_matched_SP_v1.0: pipeline: LF - subdir: CFIS/v1.0/matched_LF_SP + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + ls: dashdot colour: magenta marker: ^ @@ -1372,7 +1284,8 @@ LF_matched_SP_v1.0: SP_v1.0_LFmask_4k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dashdot colour: blue marker: h @@ -1404,7 +1317,8 @@ SP_v1.0_LFmask_4k: SP_v1.0_LFmask_8k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: blue marker: h @@ -1438,7 +1352,8 @@ SP_v1.0_LFmask_8k: SP_v1.3_LFmask_4k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dashdot colour: green marker: h @@ -1470,7 +1385,8 @@ SP_v1.3_LFmask_4k: SP_v1.4-P3_LFmask: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask/P3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 + ls: dashdot colour: black marker: h @@ -1513,7 +1429,8 @@ SP_v1.4-P3_LFmask: SP_v1.3_LFmask_8k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: green marker: h @@ -1561,7 +1478,8 @@ SP_v1.3_LFmask_8k: SP_v1.3_LFmask_8k_no_alpha: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: blueviolet marker: h @@ -1609,7 +1527,8 @@ SP_v1.3_LFmask_8k_no_alpha: SP_v1.3_LFmask_8k_li_2024: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: deepskyblue marker: h @@ -1657,7 +1576,8 @@ SP_v1.3_LFmask_8k_li_2024: SP_v1.3_LFmask_8k_SN8: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: darkgreen marker: h @@ -1704,7 +1624,8 @@ SP_v1.3_LFmask_8k_SN8: SP_v1.3_LFmask_8k_SN7: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: purple marker: h @@ -1751,7 +1672,8 @@ SP_v1.3_LFmask_8k_SN7: SP_v1.3_LFmask_8k_F2: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: dotted colour: lawngreen marker: h @@ -1798,7 +1720,8 @@ SP_v1.3_LFmask_8k_F2: SP_v1.4_LFmask_8k: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: solid colour: black marker: d @@ -1845,7 +1768,8 @@ SP_v1.4_LFmask_8k: SP_v1.4_LFmask_8k_noalpha: pipeline: SP - subdir: CFIS/v1.0/SP_LFmask + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + ls: solid colour: brown marker: d @@ -1892,7 +1816,8 @@ SP_v1.4_LFmask_8k_noalpha: SP_matched_MP_v1.0: pipeline: SP - subdir: CFIS/v1.0/matched_SP_MP + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_SP_MP + ls: dashdot colour: lightblue marker: D @@ -1918,7 +1843,8 @@ SP_matched_LF_fp_v1.0: SP_axel_v0.0: pipeline: SP - subdir: CFIS/v0.0 + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + ls: dotted colour: cyan marker: > @@ -1963,7 +1889,8 @@ SP_axel_v0.0: SP_v0.1.1: pipeline: SP - subdir: CFIS/v0.0 + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + ls: dashdot colour: cyan marker: v @@ -1997,7 +1924,8 @@ SP_v0.1.1: DES: pipeline: SP - subdir: DES + subdir: /n17data/mkilbing/astro/data/DES + ls: solid colour: orange marker: "*" diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index 3e2aa63..3513e22 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -19,11 +19,11 @@ # %% cv = CosmologyValidation( - versions=["SP_v1.4.5"], + versions=["SP_v1.4.5_leak_corr", "SP_v1.4.6_leak_corr", "SP_v1.4.7_leak_corr"], data_base_dir="/n17data/mkilbing/astro/data", - npatch=1, - theta_min=1, - theta_max=250, + npatch=256, + theta_min=1.0, + theta_max=250.0, nbins=20, theta_min_plot=1, theta_max_plot=250, diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index c57b833..78a9d89 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,5 +1,7 @@ # %% +import copy import os +from pathlib import Path import colorama import healpy as hp @@ -25,7 +27,6 @@ class CosmologyValidation: def __init__( self, versions, - data_base_dir, catalog_config="./cat_config.yaml", output_dir=None, rho_tau_method="lsq", @@ -49,10 +50,8 @@ def __init__( pol_factor=True, nrandom_cell=10, cosmo_params=None, - redshift_file=None + redshift_file=None, ): - self.versions = versions - self.data_base_dir = data_base_dir self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method self.compute_cov_rho = compute_cov_rho @@ -99,15 +98,54 @@ def __init__( with open(catalog_config, "r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) - for ver in ["nz", *versions]: - if ver not in cc: - raise KeyError( - f"Version string {ver} not found in config file{catalog_config}" - ) - version_base = f"{data_base_dir}/{cc[ver]['subdir']}" + def resolve_paths_for_version(ver): + """Resolve relative paths for a version using its subdir.""" + subdir = Path(cc[ver]["subdir"]) for key in cc[ver]: if "path" in cc[ver][key]: - cc[ver][key]["path"] = f"{version_base}/{cc[ver][key]['path']}" + path = Path(cc[ver][key]["path"]) + cc[ver][key]["path"] = ( + str(path) if path.is_absolute() else str(subdir / path) + ) + + resolve_paths_for_version("nz") + processed = {"nz"} + final_versions = [] + + for ver in versions: + if ver.endswith("_leak_corr"): + base_ver = ver.replace("_leak_corr", "") + target = base_ver + if base_ver not in cc: + raise KeyError( + f"Base version {base_ver} not found for {ver} in config file " + f"{catalog_config}" + ) + if "e1_col_corrected" not in cc[base_ver]["shear"]: + raise ValueError( + f"{base_ver} does not have e1_col_corrected/e2_col_corrected " + f"fields; cannot create {ver}" + ) + else: + target = ver + if ver not in cc: + raise KeyError( + f"Version string {ver} not found in config file " + f"{catalog_config}" + ) + + if target not in processed: + resolve_paths_for_version(target) + processed.add(target) + + if ver.endswith("_leak_corr"): + cc[ver] = copy.deepcopy(cc[base_ver]) + cc[ver]["shear"]["e1_col"] = cc[base_ver]["shear"]["e1_col_corrected"] + cc[ver]["shear"]["e2_col"] = cc[base_ver]["shear"]["e2_col_corrected"] + + final_versions.append(ver) + + self.versions = final_versions # Override output directory if provided if output_dir is not None: @@ -1927,7 +1965,6 @@ def plot_cosebis( plot_cosebis_covariance_matrix, plot_cosebis_modes, plot_cosebis_scale_cut_heatmap, - scale_cut_to_bins, ) # Use instance defaults if not specified @@ -1937,8 +1974,8 @@ def plot_cosebis( # Determine variance method based on whether theoretical covariance is used var_method = "analytic" if cov_path is not None else "jackknife" - - # Create output filename with integration parameters to match Snakemake expectations + + # Create output filename with integration parameters to match Snakemake out_stub = ( f"{output_dir}/{version}_cosebis_minsep={min_sep_int}_" f"maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}_" @@ -1976,7 +2013,9 @@ def plot_cosebis( # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use # full range if fiducial_scale_cut is not None: - plot_results = results[find_conservative_scale_cut_key(results, fiducial_scale_cut)] + plot_results = results[ + find_conservative_scale_cut_key(results, fiducial_scale_cut) + ] else: # Use full range result (largest scale cut) max_range_key = max(results.keys(), key=lambda x: x[1] - x[0]) @@ -2010,7 +2049,9 @@ def plot_cosebis( "max_sep": max_sep or self.treecorr_config["max_sep"], "nbins": nbins or self.treecorr_config["nbins"], } - gg_temp = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_temp) + gg_temp = self.calculate_2pcf( + version, npatch=npatch, **treecorr_config_temp + ) plot_cosebis_scale_cut_heatmap( results, @@ -2053,9 +2094,10 @@ def calculate_pseudo_cl_eb_cov(self): self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") lmax = 2 * self.nside - path_redshift_distr = ( - self.data_base_dir + self.cc[ver]["shear"]["redshift_distr"] - ) + redshift_path = Path(self.cc[ver]["shear"]["redshift_distr"]) + if not redshift_path.is_absolute(): + redshift_path = Path(self.cc[ver]["subdir"]) / redshift_path + path_redshift_distr = str(redshift_path) pw = hp.pixwin(nside, lmax=lmax) # Load redshift distribution and calculate theory C_ell diff --git a/src/sp_validation/tests/test_catalog_paths.py b/src/sp_validation/tests/test_catalog_paths.py new file mode 100644 index 0000000..a66778c --- /dev/null +++ b/src/sp_validation/tests/test_catalog_paths.py @@ -0,0 +1,97 @@ +"""Lightweight checks that catalog paths referenced in ``cat_config.yaml`` exist. + +These tests do not load the data; they only ensure that the configuration points +to readable files so that downstream validation runs fail fast if a path drifts. +""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import Dict, Iterator, Tuple + +import yaml + + +CATALOG_CONFIG = ( + Path(__file__).resolve().parents[3] / "notebooks" / "cosmo_val" / "cat_config.yaml" +) + +EXPECTED_MISSING = { + "SP_v1.0": {"shear", "psf"}, + "SP_v1.1": {"shear", "psf"}, + "SP_v1.4": {"shear", "star", "psf"}, + "SP_v1.4_conv": {"shear", "star", "psf"}, + "SP_v1.4_noalpha": {"shear", "star", "psf"}, + "SP_v1.4-P1+3": {"shear", "star", "psf"}, + "SP_v1.4-P1+3_wcs": {"shear", "star", "psf"}, + "SP_v1.4-P1+3_no_alpha": {"shear", "star", "psf"}, + "SP_v1.4-P1+3_li_2024": {"shear", "star", "psf"}, + "SP_v1.4-P1+3+4": {"shear", "star", "psf"}, + "SP_v1.4-P1+3+4_wcs": {"shear", "star", "psf"}, + "SP_v1.4-P1+3+4_no_alpha": {"shear", "star", "psf"}, + "SP_matched_MP_v1.0": {"shear"}, +} + + +def _resolve(base: Path, candidate: str) -> Path: + """Return an absolute path given a base directory and a candidate string.""" + candidate_path = Path(candidate) + return candidate_path if candidate_path.is_absolute() else base / candidate_path + + +def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: + """Yield (name, entry) pairs for catalog-like entries in the config.""" + for name, entry in config.items(): + if not isinstance(entry, dict): + continue + if "subdir" not in entry: + continue + yield name, entry + + +def test_catalog_files_exist(): + """Ensure every shear/star/psf entry references an on-disk file.""" + config = yaml.safe_load(CATALOG_CONFIG.read_text()) + + missing = defaultdict(set) + + for name, entry in _iter_catalog_entries(config): + base = Path(entry["subdir"]) + assert base.is_absolute(), f"{name}: subdir must be absolute ({base})" + + if name == "nz": + dndz_path = _resolve(base, entry["dndz"]["path"]) + assert ( + dndz_path.parent.is_dir() + ), f"{name}: dndz parent directory missing ({dndz_path.parent})" + continue + + for block_name in ("shear", "star", "psf"): + block = entry.get(block_name) + if not block: + continue + resolved_path = _resolve(base, block["path"]) + if not resolved_path.is_file(): + missing[name].add(block_name) + + unexpected = { + name: blocks - EXPECTED_MISSING.get(name, set()) + for name, blocks in missing.items() + if blocks - EXPECTED_MISSING.get(name, set()) + } + assert not unexpected, ( + "Unexpected missing catalog files detected: " + f"{ {name: sorted(blocks) for name, blocks in unexpected.items()} }" + ) + + resolved = { + name: expected - missing.get(name, set()) + for name, expected in EXPECTED_MISSING.items() + if expected - missing.get(name, set()) + } + assert not resolved, ( + "Previously missing catalog files are now available; " + f"update EXPECTED_MISSING to reflect this change: " + f"{ {name: sorted(blocks) for name, blocks in resolved.items()} }" + ) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py new file mode 100644 index 0000000..55b8f80 --- /dev/null +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -0,0 +1,127 @@ +"""UNIT TESTS FOR COSMOLOGY VALIDATION CLASS. + +This module contains integration tests for the CosmologyValidation class, +specifically testing the ellipticity_suffix parameter functionality for +handling leak-corrected ellipticity columns. + +:Author: cdaley + +""" + +import os + +import pytest + +from sp_validation.cosmo_val import CosmologyValidation + + +class TestCosmologyValidation: + """Test CosmologyValidation initialization and additive bias calculation.""" + + @pytest.fixture + def base_config(self, tmp_path): + """Common configuration parameters for tests.""" + # Get the path to the repo root (3 levels up from this test file) + repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + ) + catalog_config = os.path.join( + repo_root, "notebooks", "cosmo_val", "cat_config.yaml" + ) + + # Use temporary directory for outputs + output_dir = tmp_path / "test_output" + output_dir.mkdir() + + return { + "catalog_config": catalog_config, + "output_dir": str(output_dir), + "npatch": 1, + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + } + + @pytest.mark.parametrize( + "version,e1_col,e2_col", + [ + ("SP_v1.4.5", "e1", "e2"), + ("SP_v1.4.6", "e1", "e2"), + ("SP_v1.4.5_glass_mock", "e1", "e2"), + ("SP_v1.4.5_bright", "e1", "e2"), + ("SP_v1.4.5_faint", "e1", "e2"), + ("SP_v1.4.5_intermediate", "e1", "e2"), + ("SP_v1.4.5.A", "g1", "g2"), + ("SP_v1.4.7", "e1", "e2"), + ("SP_v1.4.8", "e1", "e2"), + ], + ) + def test_additive_bias_base_columns(self, base_config, version, e1_col, e2_col): + """Test additive bias calculation using base ellipticity columns. + + This test initializes CosmologyValidation without an ellipticity_suffix, + which means it will use the default columns defined in the catalog + configuration. + """ + cv = CosmologyValidation( + versions=[version], + **base_config, + ) + + # Verify version names remain unchanged + assert cv.versions == [version] + + # Verify the ellipticity columns are the base columns + assert cv.cc[version]["shear"]["e1_col"] == e1_col + assert cv.cc[version]["shear"]["e2_col"] == e2_col + + # Calculate additive bias + cv.calculate_additive_bias() + + # Verify c1 and c2 were calculated and stored + assert hasattr(cv, "_c1") and hasattr(cv, "_c2") + assert version in cv.c1 + assert version in cv.c2 + + # Verify the values are numeric (not NaN or None) + assert isinstance(cv.c1[version], float) + assert isinstance(cv.c2[version], float) + + @pytest.mark.parametrize("version", ["SP_v1.4.5"]) + def test_additive_bias_leak_corrected_columns(self, base_config, version): + """Test additive bias calculation using leak-corrected columns. + + This test requests a leak-corrected version by passing "{version}_leak_corr" + as the version name. The CosmologyValidation class automatically detects + the _leak_corr suffix and creates the config entry using e1_col_corrected + and e2_col_corrected from the base version. + """ + version_leak_corr = f"{version}_leak_corr" + + cv = CosmologyValidation( + versions=[version_leak_corr], + **base_config, + ) + + # Verify version names include the _leak_corr suffix + assert cv.versions == [version_leak_corr] + + # Verify the leak-corrected config was auto-created with corrected columns + assert cv.cc[version_leak_corr]["shear"]["e1_col"] == "e1_leak_corrected" + assert cv.cc[version_leak_corr]["shear"]["e2_col"] == "e2_leak_corrected" + + # Verify original config entry remains unchanged + assert cv.cc[version]["shear"]["e1_col"] == "e1" + assert cv.cc[version]["shear"]["e2_col"] == "e2" + + # Calculate additive bias + cv.calculate_additive_bias() + + # Verify c1 and c2 were calculated and stored with leak-corrected name + assert hasattr(cv, "_c1") and hasattr(cv, "_c2") + assert version_leak_corr in cv.c1 + assert version_leak_corr in cv.c2 + + # Verify the values are numeric + assert isinstance(cv.c1[version_leak_corr], float) + assert isinstance(cv.c2[version_leak_corr], float) From 62868c0777a894dc7e4d1f25b737249c3f62ec73 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Oct 2025 00:17:34 +0200 Subject: [PATCH 04/29] gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index df94531..09e4ece 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ notebooks/demo_check_footprint.ipynb notebooks/demo_calibrate_minimal_cat.ipynb notebooks/leakage_minimal.ipynb notebooks/demo_add_bands.ipynb + +CLAUDE.md +AGENTS.md \ No newline at end of file From 0ce82ef44853ca7ca231c3e01882a6563d5f5893 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 13 Oct 2025 18:57:16 +0200 Subject: [PATCH 05/29] remove PR draft file --- docs/PR_DRAFT.md | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 docs/PR_DRAFT.md diff --git a/docs/PR_DRAFT.md b/docs/PR_DRAFT.md deleted file mode 100644 index 38b1bfe..0000000 --- a/docs/PR_DRAFT.md +++ /dev/null @@ -1,38 +0,0 @@ -# Draft PR: Catalog Path Cleanup & Guardrails - -## Summary - -- Normalised every `subdir` entry in `notebooks/cosmo_val/cat_config.yaml` so the loader relies on explicit `/n17data/...` locations rather than the old `data_dir` fallback. Shared resources (redshift distributions, common star/psf FITS) now point at their canonical mounts. -- Updated the Pseudo-`C_\ell` code path in `src/sp_validation/cosmo_val.py` to read redshift files directly from the catalog configuration, removing the `data_base_dir` assumption. -- Added `src/sp_validation/tests/test_catalog_paths.py`, a lightweight validator that resolves each shear/star/psf reference and records a small allow-list for catalogues whose source files are currently unavailable. -- Kept the additive-bias regression test focused on the catalogues that still have data on disk; the missing releases stay excluded so CI remains green. - -## Outstanding Data Gaps - -All of the catalogues below are missing their **shear** FITS on the shared mounts (star/psf often missing as well): - -1. `SP_v1.0`, `SP_v1.1`, and `SP_matched_MP_v1.0` -2. The `SP_v1.4` family: `SP_v1.4`, `SP_v1.4_conv`, `SP_v1.4_noalpha` -3. Every `SP_v1.4-P1+3*` derivative (same missing 2022-era inputs as item 2) - -These entries remain in `cat_config.yaml` for provenance, are listed in the validator allow-list, and are intentionally omitted from the additive-bias parameter set. The follow-up PR should either restore their data to `/n17data/...`, retarget them at new releases (e.g. the 2024 v1.4.1 products), or mark them as deprecated. - -## Proposed Next Steps - -1. **Decide The Fate Of The Legacy Catalogues** - - Recover the missing FITS/parquet files, or formally deprecate the versions above (update docs/tests accordingly). - -2. **Re-home The v1.4 Variants (if data exist)** - - Point each affected block at the correct `/n17data/UNIONS/WL/v1.4.x//` directory. - - Prefer catalogue-local star/psf fits when the directory is more than a symlink; fall back to the shared root otherwise. - - Remove the entries from `EXPECTED_MISSING` and re-enable them in regression tests once their files resolve. - -3. **Document The Conventions** - - Add a short developer note describing the absolute-path policy, preferred star/psf strategy, and how to run the new path validator before pushing catalog edits. - - Optionally list the currently offline datasets so collaborators know what needs recovering. - -## Validation Checklist - -- `app python -m pytest src/sp_validation/tests/test_catalog_paths.py` -- `app python -m pytest src/sp_validation/tests/test_cosmo_val.py -k additive_bias` -- Manual `ls` spot-check of any retargeted catalogue directories to ensure we pick up the intended local star/psf files rather than parent symlinks. From 6a44359650d9a66b1426ef6c2c844d8f12b319c1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 13 Oct 2025 19:05:27 +0200 Subject: [PATCH 06/29] propagate changes to run_cosmo_val --- notebooks/cosmo_val/run_cosmo_val.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index 3513e22..037a38d 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -20,8 +20,7 @@ # %% cv = CosmologyValidation( versions=["SP_v1.4.5_leak_corr", "SP_v1.4.6_leak_corr", "SP_v1.4.7_leak_corr"], - data_base_dir="/n17data/mkilbing/astro/data", - npatch=256, + npatch=1, theta_min=1.0, theta_max=250.0, nbins=20, From 746479b54264677a38127cd4b7af5fa69c8471d9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 15 Oct 2025 00:11:54 +0200 Subject: [PATCH 07/29] Prune legacy catalogs and tighten survey stats --- notebooks/cosmo_val/cat_config.yaml | 3276 +++++++---------- notebooks/cosmo_val/compute_theory_cov.py | 4 +- src/sp_validation/cosmo_val.py | 132 +- src/sp_validation/tests/test_catalog_paths.py | 38 +- 4 files changed, 1487 insertions(+), 1963 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 493b5db..dd02dfb 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1,1971 +1,1397 @@ -paths: - output: ./output +DES: + subdir: /n17data/mkilbing/astro/data/DES + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: '*' + cov_th: + A: 4143 + n_e: 5.67 + n_psf: 3.8 + sigma_e: 0.29 + path_rho: rho_stats_DES.fits + path_tau: tau_stats_DES.fits + path_xi_plus: xi_plus_DES.fits + path_xi_minus: xi_minus_DES.fits + psf: + PSF_size: piff_T + square_size: false + star_size: obs_T + hdu: 1 + path: psf_y3a1-v29.fits + ra_col: ra + dec_col: dec + e1_PSF_col: piff_e1 + e1_star_col: obs_e1 + e2_PSF_col: piff_e2 + e2_star_col: obs_e2 + shear: + R: 1.0 + R11: R11 + R22: R22 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: DES_Y3_cut.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_psf + e2_col: e2 + e2_PSF_col: e2_psf + label: DES-Y3 + star: + ra_col: ra + dec_col: dec + e1_col: obs_e1 + e2_col: obs_e2 + path: psf_y3a1-v29.fits -nz: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz +LF_matched_SP_v1.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + pipeline: LF + colour: magenta + getdist_colour: 0.0, 0.5, 0.0 + ls: dashdot + marker: ^ + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt + path: masked_lensfit_goldshape_2022v1.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: full_starcat_tmp_1_10_seed_1234.fits - dndz: - path: dndz - blind: A +LF_v1.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit + pipeline: LF + colour: r + getdist_colour: 1.0, 0.5, 0.0 + ls: solid + marker: '8' + shear: + R: 1.0 + covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt + path: lensfit_goldshape_2022v1.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: full_starcat_tmp_1_10_seed_1234.fits -SP_test: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +LF_v2.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit + pipeline: LF + colour: r + ls: dashed + marker: p + shear: + R: 1.0 + path: lensfit_goldshape_2022v2_psf1.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1_psf + e2_col: e2_psf + path: Pall_hdu1_THELI_psfs1_mosaic.fits - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_axel_v0.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dotted + marker: '' + cov_th: + A: 1500 + n_e: 6.47 + n_psf: 0.47 + sigma_e: 0.3 + path_rho: rho_stats_SP_v0.0.fits + path_tau: tau_stats_SP_v0.0.fits + path_xi_plus: xi_plus_SP_v0.0.fits + path_xi_minus: xi_minus_SP_v0.0.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: star_cat.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: shapepipe_1500_goldshape_v1.fits + w_col: w + e1_col: g1 + e1_PSF_col: E1_PSF_HSM + e2_col: g2 + e2_PSF_col: E2_PSF_HSM + star: + ra_col: RA + dec_col: DEC + e1_col: E1_STAR_HSM + e2_col: E2_STAR_HSM + path: star_cat.fits -SP_v1.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_matched_LF_v1.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP + pipeline: SP + colour: magenta + getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 + ls: dotted + marker: v + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt + path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ls: solid - colour: blue - marker: o - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_2022_v1.0.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" +SP_test: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.1.fits -SP_v1.1: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v0.1.1: + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: v + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: star_cat.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: DEC + e1_col: E1_STAR_HSM + e2_col: E2_STAR_HSM + path: star_cat.fits - ls: dashed - colour: b - marker: s - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_2022_v1.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" +SP_v1.0_LFmask_4k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: blue + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dashdot + marker: h + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_2022_v1.0_mtheli4k.fits + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits -SP_v1.3: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.0_LFmask_8k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: blue + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ls: solid - colour: green - marker: p - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_extended_2022_v1.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: p + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_v1.3.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3.fits -SP_v1.4: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.3_LFmask_4k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: green + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dashdot + marker: h + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_2022_v1.3_mtheli4k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3_LFmask_8k: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: green + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 7.6 + n_psf: 0.68 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits -SP_v1.4_conv: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.3_LFmask_8k_F2: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: lawngreen + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 9.4 + n_psf: 0.48 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits + label: SP_LFmask_star - ls: solid - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3_LFmask_8k_SN7: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: purple + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 10.989 + n_psf: 0.48 + sigma_e: 0.3796 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits + label: SP_LFmask_star -SP_v1.4_noalpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe +SP_v1.3_LFmask_8k_SN8: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: darkgreen + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 9.7 + n_psf: 0.48 + sigma_e: 0.35 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits + label: SP_LFmask_star - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.3_LFmask_8k_li_2024: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: deepskyblue + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 7.6 + n_psf: 0.68 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits + w_col: w + e1_col: e1_cor_2 + e1_PSF_col: e1_PSF + e2_col: e2_cor_2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits +SP_v1.3_LFmask_8k_no_alpha: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: blueviolet + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dotted + marker: h + cov_th: + A: 2138 + n_e: 7.6 + n_psf: 0.68 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.3_LFmask_8k.fits + path_tau: tau_stats_SP_v1.3_LFmask_8k.fits + path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits + path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: false + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + R11: 1.0 + R22: 1.0 + path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits + w_col: w + e1_col: e1_cor + e1_PSF_col: e1_PSF + e2_col: e2_cor + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits SP_v1.4-P3: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 337 #deg^2 - n_e: 7.45 #arcmin-2 - n_psf: 0.34 #arcmin-2 - sigma_e: 0.32 - path_rho: rho_stats_SP_v1.4-P3.fits - path_tau: tau_stats_SP_v1.4-P3.fits - path_xi_plus: xi_plus_SP_v1.4-P3.fits - path_xi_minus: xi_minus_SP_v1.4-P3.fits - shear: - path: unions_shapepipe_extended_2022_P3_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P1+3.fits - path_tau: tau_stats_SP_v1.4-P1+3.fits - path_xi_plus: xi_plus_SP_v1.4-P1+3.fits - path_xi_minus: xi_minus_SP_v1.4-P1+3.fits - shear: - path: unions_shapepipe_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P1+3_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_wcs: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_no_alpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 + pipeline: SP + colour: brown + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 337 + n_e: 7.45 + n_psf: 0.34 + sigma_e: 0.32 + path_rho: rho_stats_SP_v1.4-P3.fits + path_tau: tau_stats_SP_v1.4-P3.fits + path_xi_plus: xi_plus_SP_v1.4-P3.fits + path_xi_minus: xi_minus_SP_v1.4-P3.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 2 + path: unions_shapepipe_psf_2022_P3_v1.4.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_P3_v1.4.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_P3_v1.4.fits - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_rmalpha_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_li_2024: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3 - - ls: dashed - colour: gold - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_rmalpha_li_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 - - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4_wcs: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 - - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_extended_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4_no_alpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P1+3+4 - - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_extended_rmalpha_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.2: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 - - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.2.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4-P3_LFmask: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 + pipeline: SP + colour: black + getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 + ls: dashdot + marker: h + cov_th: + A: 223 + n_e: 7.44 + n_psf: 0.32 + sigma_e: 0.31 + path_rho: rho_stats_SP_v1.4-P3_LFmask.fits + path_tau: tau_stats_SP_v1.4-P3_LFmask.fits + path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits + path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_2022_v1.4_mtheli4k.fits + w_col: w + e1_col: e1 + e2_col: e2 + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits SP_v1.4.1_noleakage: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 3191.1827474082734 + n_e: 7.0582373812033286 + n_psf: 0.5705736293858423 + sigma_e: 0.317469332009887 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_2024_v1.4.1.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.1.fits - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3191.1827474082734 #deg^2 - n_e: 7.0582373812033286 #arcmin-2 - n_psf: 0.5705736293858423 #arcmin-2 - sigma_e: 0.317469332009887 - shear: - path: unions_shapepipe_2024_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4.2: + subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 1826 + n_e: 7.75 + n_psf: 0.35 + sigma_e: 0.3133 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.2.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: dodgerblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: dodgerblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2582.310451583689 + n_e: 7.748708180391998 + n_psf: 0.4973044366748657 + sigma_e: 0.4375145387558129 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.5_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.5_glass_mock: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: crimson - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4.5.A: + subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 + pipeline: SP + colour: red + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: shapepipe_SPv1.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: g1 + e1_PSF_col: e1_PSF + e2_col: g2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_bright: - pipeline: SP - subdir: /n17data/murray/unions_cats - - ls: dashed - colour: mediumblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: mediumblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_faint: - pipeline: SP - subdir: /n17data/murray/unions_cats + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: forestgreen + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ls: dashed - colour: forestgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True +SP_v1.4.5_glass_mock: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: crimson + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2782 + n_e: 7.18 + n_psf: 0.752316232272063 + sigma_e: 0.4370966656902571 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_intermediate: - pipeline: SP - subdir: /n17data/murray/unions_cats - - ls: dashed - colour: orange - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5.A: - pipeline: SP - subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 - - ls: dashed - colour: red - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: shapepipe_SPv1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: g1 - e2_col: g2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.6: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - shouldn't change with size cuts - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.7: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: darkorchid - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: null # TBD - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkorchid + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2050.5970269635545 + n_e: 6.121543643587931 + n_psf: 0.752316232272063 + sigma_e: 0.3797419081703805 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.7_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.8: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - - ls: dashed - colour: darkorchid - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: null # TBD - n_e: null # TBD - n_psf: null # TBD - sigma_e: null # TBD - shear: - path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_col_corrected: e1_leak_corrected - e2_col_corrected: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.5.4: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.5.x - - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.5.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.5.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -LF_v1.0: - pipeline: LF - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit - - ls: solid - colour: r - marker: "8" - getdist_colour: 1.0, 0.5, 0.0 - shear: - path: lensfit_goldshape_2022v1.fits - covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: full_starcat_tmp_1_10_seed_1234.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - -LF_v2.0: - pipeline: LF - subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit - - ls: dashed - colour: r - marker: p - shear: - path: lensfit_goldshape_2022v2_psf1.fits - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: Pall_hdu1_THELI_psfs1_mosaic.fits - ra_col: RA - dec_col: Dec - e1_col: e1_psf - e2_col: e2_psf - -SP_matched_LF_v1.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - - ls: dotted - colour: magenta - marker: v - getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 - shear: - path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -LF_matched_SP_v1.0: - pipeline: LF - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - - ls: dashdot - colour: magenta - marker: ^ - getdist_colour: 0.0, 0.5, 0.0 - shear: - path: masked_lensfit_goldshape_2022v1.fits - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: full_starcat_tmp_1_10_seed_1234.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - -SP_v1.0_LFmask_4k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dashdot - colour: blue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_2022_v1.0_mtheli4k.fits - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.0_LFmask_8k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: blue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.3_LFmask_4k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dashdot - colour: green - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_2022_v1.3_mtheli4k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.4-P3_LFmask: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 - - ls: dashdot - colour: black - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 223 #deg^2 - n_e: 7.44 #arcmin-2 - n_psf: 0.32 #arcmin-2 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P3_LFmask.fits - path_tau: tau_stats_SP_v1.4-P3_LFmask.fits - path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits - path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits - shear: - path: unions_shapepipe_2022_v1.4_mtheli4k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.3_LFmask_8k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: green - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_no_alpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: blueviolet - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_li_2024: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: deepskyblue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_SN8: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: darkgreen - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 9.7 - n_psf: 0.48 - sigma_e: 0.35 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_SN7: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: purple - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 10.989 - n_psf: 0.48 - sigma_e: 0.3796 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_F2: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: dotted - colour: lawngreen - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 9.4 - n_psf: 0.48 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkorchid + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2049.034211379564 + n_e: 6.121484916053109 + n_psf: 0.752316232272063 + sigma_e: 0.37973978258052915 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.8_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4_LFmask_8k: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: solid - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2137.7977618140676 #deg^2 - n_e: 7.976462506484096 #arcmin-2 - n_psf: 0.5434016250405327 #arcmin-2 - sigma_e: 0.31509572849714534 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: d + cov_th: + A: 2137.7977618140676 + n_e: 7.976462506484096 + n_psf: 0.5434016250405327 + sigma_e: 0.31509572849714534 + path_rho: rho_stats_SP_v1.4.fits + path_tau: tau_stats_SP_v1.4.fits + path_xi_plus: xi_plus_SP_v1.4.fits + path_xi_minus: xi_minus_SP_v1.4.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits SP_v1.4_LFmask_8k_noalpha: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2137.7977618140676 #deg^2 - n_e: 7.976462506484096 #arcmin-2 - n_psf: 0.5434016250405327 #arcmin-2 - sigma_e: 0.31509572849714534 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_matched_MP_v1.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_SP_MP - - ls: dashdot - colour: lightblue - marker: D - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: cfis-shapepipe.parquet - e1_col: e1 - e2_col: e2 - R: 1.0 - -SP_matched_LF_fp_v1.0: - pipeline: SP - ls: dashed - colour: lightblue - marker: < - getdist_colour: 0.08130806428547999, 0.1463709449188496, 0.674947827197815 - shear: - path: matched_footprint_shapepipe.fits - covmat_file: ./covs/lensfit_matched/cov_lensfit_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - -SP_axel_v0.0: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: brown + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: d + cov_th: + A: 2137.7977618140676 + n_e: 7.976462506484096 + n_psf: 0.5434016250405327 + sigma_e: 0.31509572849714534 + path_rho: rho_stats_SP_v1.4.fits + path_tau: tau_stats_SP_v1.4.fits + path_xi_plus: xi_plus_SP_v1.4.fits + path_xi_minus: xi_minus_SP_v1.4.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits + w_col: w + e1_col: e1_cor + e1_PSF_col: e1_PSF + e2_col: e2_cor + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ls: dotted - colour: cyan - marker: > - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1500 - n_e: 6.47 - n_psf: 0.47 - sigma_e: 0.30 - path_rho: rho_stats_SP_v0.0.fits - path_tau: tau_stats_SP_v0.0.fits - path_xi_plus: xi_plus_SP_v0.0.fits - path_xi_minus: xi_minus_SP_v0.0.fits - shear: - path: shapepipe_1500_goldshape_v1.fits - e1_col: g1 - e2_col: g2 - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - w_col: w - R: 1.0 - star: - path: star_cat.fits - ra_col: RA - dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM - psf: - path: star_cat.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - square_size: True - -SP_v0.1.1: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 - - ls: dashdot - colour: cyan - marker: v - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - star: - path: star_cat.fits - ra_col: RA - dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM - psf: - path: star_cat.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM +SP_v1.5.4: + subdir: /n17data/UNIONS/WL/v1.5.x + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.5.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.5.a.fits -DES: - pipeline: SP - subdir: /n17data/mkilbing/astro/data/DES +nz: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + dndz: + blind: A + path: dndz - ls: solid - colour: orange - marker: "*" - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 4143 - n_e: 5.67 - n_psf: 3.8 - sigma_e: 0.29 - path_rho: rho_stats_DES.fits - path_tau: tau_stats_DES.fits - path_xi_plus: xi_plus_DES.fits - path_xi_minus: xi_minus_DES.fits - shear: - path: DES_Y3_cut.fits - label: DES-Y3 - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_psf - e2_PSF_col: e2_psf - w_col: w - R: 1.0 - R11: R11 - R22: R22 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - star: - path: psf_y3a1-v29.fits - ra_col: ra - dec_col: dec - e1_col: obs_e1 - e2_col: obs_e2 - psf: - path: psf_y3a1-v29.fits - hdu: 1 - ra_col: ra - dec_col: dec - e1_PSF_col: piff_e1 - e2_PSF_col: piff_e2 - e1_star_col: obs_e1 - e2_star_col: obs_e2 - PSF_size: piff_T - star_size: obs_T - square_size: False +paths: + output: ./output diff --git a/notebooks/cosmo_val/compute_theory_cov.py b/notebooks/cosmo_val/compute_theory_cov.py index 2c93d47..8c20a73 100644 --- a/notebooks/cosmo_val/compute_theory_cov.py +++ b/notebooks/cosmo_val/compute_theory_cov.py @@ -41,7 +41,7 @@ def get_params_rho_tau(cat, survey="other"): base_dir = '/home/guerrini/data/' - versions = ['SP_v1.4-P3', 'SP_v1.4-P3_LFmask', 'SP_v1.4-P1+3', 'SP_v1.3_LFmask_8k', 'SP_axel_v0.0', 'DES'] + versions = ['SP_v1.4-P3', 'SP_v1.4-P3_LFmask', 'SP_v1.3_LFmask_8k', 'SP_axel_v0.0', 'DES'] path_config = '/home/guerrini/sp_validation/notebooks/cosmo_val/cat_config.yaml' output_dir = '/home/guerrini/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/' @@ -94,4 +94,4 @@ def get_params_rho_tau(cat, survey="other"): print("--- Covariance computation %s seconds ---" % (time.time() - start_time)) np.save(output_dir+'/cov_tau_'+ver+'_th.npy', cov) - print("Saved covariance matrix of version: ", ver) \ No newline at end of file + print("Saved covariance matrix of version: ", ver) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 78a9d89..4527532 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -95,7 +95,8 @@ def __init__( "cross_patch_weight": "match" if var_method == "jackknife" else "simple", } - with open(catalog_config, "r") as file: + self.catalog_config_path = Path(catalog_config) + with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) def resolve_paths_for_version(ver): @@ -154,6 +155,135 @@ def resolve_paths_for_version(ver): if not os.path.exists(cc["paths"]["output"]): os.mkdir(cc["paths"]["output"]) + def compute_survey_stats( + self, + ver, + weights_key_override=None, + mask_path=None, + nside=None, + overwrite_config=False, + ): + """Compute effective survey statistics for a catalog version. + + Parameters + ---------- + ver : str + Version string registered in the catalog config. + weights_key_override : str, optional + Override the weight column key (defaults to the configured `w_col`). + mask_path : str, optional + Explicit mask path to use when measuring survey area. + nside : int, optional + If provided, compute survey area from the catalog using this NSIDE when no + mask path is available. + overwrite_config : bool, optional + If True, persist the derived statistics back to the catalog configuration. + + Returns + ------- + dict + Dictionary containing: + - area_deg2: Survey area in square degrees. + - n_eff: Effective number density per arcmin^2. + - sigma_e: Per-component shape noise. + - sum_w: Sum of weights. + - sum_w2: Sum of squared weights. + - catalog_size: Number of galaxies processed. + """ + if ver not in self.cc: + raise KeyError(f"Version {ver} not found in catalog configuration") + + shear_cfg = self.cc[ver]["shear"] + cov_th = self.cc[ver].get("cov_th", {}) + + if "path" not in shear_cfg: + raise KeyError(f"No shear catalog path defined for version {ver}") + + catalog_path = shear_cfg["path"] + if not os.path.exists(catalog_path): + raise FileNotFoundError(f"Shear catalog not found: {catalog_path}") + + data = fits.getdata(catalog_path, memmap=True) + n_rows = len(data) + + e1 = np.asarray(data[shear_cfg["e1_col"]], dtype=float) + e2 = np.asarray(data[shear_cfg["e2_col"]], dtype=float) + + weight_column = weights_key_override or shear_cfg["w_col"] + if weight_column not in data.columns.names: + raise KeyError(f"Weight column '{weight_column}' missing in {catalog_path}") + + w = np.asarray(data[weight_column], dtype=float) + + sum_w = float(np.sum(w)) + sum_w2 = float(np.sum(w**2)) + sum_w2_e2 = float(np.sum((w**2) * (e1**2 + e2**2))) + + if mask_path is not None: + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask path not found: {mask_path}") + mask_candidate = mask_path + else: + mask_candidate = self.cc[ver].get("mask") + if isinstance(mask_candidate, str) and not os.path.isabs(mask_candidate): + mask_candidate = str(Path(self.cc[ver]["subdir"]) / mask_candidate) + if mask_candidate is not None and not os.path.exists(mask_candidate): + mask_candidate = None + + area_deg2 = None + if mask_candidate is not None and os.path.exists(mask_candidate): + area_deg2 = self._area_from_mask(mask_candidate) + elif cov_th.get("A") is not None: + area_deg2 = float(cov_th["A"]) + elif nside is not None: + area_deg2 = self._area_from_catalog(catalog_path, nside) + else: + raise ValueError( + f"Unable to determine survey area for {ver}. Provide mask_path or nside." + ) + + area_arcmin2 = area_deg2 * 3600.0 + + n_eff = (sum_w**2) / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 + sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 + + results = { + "area_deg2": area_deg2, + "n_eff": n_eff, + "sigma_e": sigma_e, + "sum_w": sum_w, + "sum_w2": sum_w2, + "catalog_size": n_rows, + } + + if overwrite_config: + if "cov_th" not in self.cc[ver]: + self.cc[ver]["cov_th"] = {} + self.cc[ver]["cov_th"]["A"] = float(area_deg2) + self.cc[ver]["cov_th"]["n_e"] = float(n_eff) + self.cc[ver]["cov_th"]["sigma_e"] = float(sigma_e) + self._write_catalog_config() + + return results + + def _area_from_catalog(self, catalog_path, nside): + data = fits.getdata(catalog_path, memmap=True) + ra = np.asarray(data["RA"], dtype=float) + dec = np.asarray(data["Dec"], dtype=float) + theta = np.radians(90.0 - dec) + phi = np.radians(ra) + pix = hp.ang2pix(nside, theta, phi, lonlat=False) + unique_pix = np.unique(pix) + return float(unique_pix.size * hp.nside2pixarea(nside, degrees=True)) + + def _area_from_mask(self, mask_map_path): + mask = hp.read_map(mask_map_path, dtype=np.float64) + return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) + + def _write_catalog_config(self): + with self.catalog_config_path.open("w") as file: + yaml.dump(self.cc, file, sort_keys=False) + def color_reset(self): print(colorama.Fore.BLACK, end="") diff --git a/src/sp_validation/tests/test_catalog_paths.py b/src/sp_validation/tests/test_catalog_paths.py index a66778c..51e4322 100644 --- a/src/sp_validation/tests/test_catalog_paths.py +++ b/src/sp_validation/tests/test_catalog_paths.py @@ -17,22 +17,6 @@ Path(__file__).resolve().parents[3] / "notebooks" / "cosmo_val" / "cat_config.yaml" ) -EXPECTED_MISSING = { - "SP_v1.0": {"shear", "psf"}, - "SP_v1.1": {"shear", "psf"}, - "SP_v1.4": {"shear", "star", "psf"}, - "SP_v1.4_conv": {"shear", "star", "psf"}, - "SP_v1.4_noalpha": {"shear", "star", "psf"}, - "SP_v1.4-P1+3": {"shear", "star", "psf"}, - "SP_v1.4-P1+3_wcs": {"shear", "star", "psf"}, - "SP_v1.4-P1+3_no_alpha": {"shear", "star", "psf"}, - "SP_v1.4-P1+3_li_2024": {"shear", "star", "psf"}, - "SP_v1.4-P1+3+4": {"shear", "star", "psf"}, - "SP_v1.4-P1+3+4_wcs": {"shear", "star", "psf"}, - "SP_v1.4-P1+3+4_no_alpha": {"shear", "star", "psf"}, - "SP_matched_MP_v1.0": {"shear"}, -} - def _resolve(base: Path, candidate: str) -> Path: """Return an absolute path given a base directory and a candidate string.""" @@ -75,23 +59,7 @@ def test_catalog_files_exist(): if not resolved_path.is_file(): missing[name].add(block_name) - unexpected = { - name: blocks - EXPECTED_MISSING.get(name, set()) - for name, blocks in missing.items() - if blocks - EXPECTED_MISSING.get(name, set()) - } - assert not unexpected, ( - "Unexpected missing catalog files detected: " - f"{ {name: sorted(blocks) for name, blocks in unexpected.items()} }" - ) - - resolved = { - name: expected - missing.get(name, set()) - for name, expected in EXPECTED_MISSING.items() - if expected - missing.get(name, set()) - } - assert not resolved, ( - "Previously missing catalog files are now available; " - f"update EXPECTED_MISSING to reflect this change: " - f"{ {name: sorted(blocks) for name, blocks in resolved.items()} }" + assert not missing, ( + "Catalog configuration references missing files: " + f"{ {name: sorted(blocks) for name, blocks in missing.items()} }" ) From 1b552381ca2fc5e63fad24835c11138e49fe5239 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 15 Oct 2025 11:41:43 +0200 Subject: [PATCH 08/29] add docstring --- src/sp_validation/cosmo_val.py | 86 ++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 4527532..e93d95b 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -24,6 +24,92 @@ # %% class CosmologyValidation: + """Framework for cosmic shear validation and systematics analysis. + + Handles two-point correlation function measurements, PSF systematics (rho/tau), + pseudo-C_ell analysis, and covariance estimation for weak lensing surveys. + Supports multiple catalog versions with automatic leakage-corrected variants. + + Parameters + ---------- + versions : list of str + Catalog version identifiers to analyze. Appending '_leak_corr' to a base + version creates a virtual catalog using leakage-corrected ellipticity columns + (e1_col_corrected/e2_col_corrected) from the base version configuration. + catalog_config : str, default './cat_config.yaml' + Path to catalog configuration YAML defining survey metadata, file paths, + and analysis settings for each version. + output_dir : str, optional + Override for output directory. If None, uses catalog config's paths.output. + rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' + Fitting method for PSF leakage systematics parameters. + cov_estimate_method : {'th', 'jk'}, default 'th' + Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife. + compute_cov_rho : bool, default True + Whether to compute covariance for rho statistics during PSF analysis. + n_cov : int, default 100 + Number of realizations for covariance estimation when using theory method. + theta_min : float, default 0.1 + Minimum angular separation in arcminutes for correlation function binning. + theta_max : float, default 250 + Maximum angular separation in arcminutes for correlation function binning. + nbins : int, default 20 + Number of angular bins for TreeCorr real-space correlation functions. + var_method : {'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'}, default 'jackknife' + TreeCorr variance estimation method. + npatch : int, default 20 + Number of spatial patches for jackknife variance estimation. + quantile : float, default 0.1587 + Quantile for uncertainty bands in plots (default: 1-sigma ≈ 0.159). + theta_min_plot : float, default 0.08 + Minimum angular scale for plotting (may differ from analysis cut). + theta_max_plot : float, default 250 + Maximum angular scale for plotting. + ylim_alpha : list of float, default [-0.005, 0.05] + Y-axis limits for alpha systematic parameter plots. + ylim_xi_sys_ratio : list of float, default [-0.02, 0.5] + Y-axis limits for xi systematics ratio plots. + nside : int, default 1024 + HEALPix resolution for pseudo-C_ell analysis and area computation. + binning : {'powspace', 'linspace', 'logspace'}, default 'powspace' + Ell binning scheme for pseudo-C_ell (powspace = ell^power spacing). + power : float, default 0.5 + Exponent for power-law binning when binning='powspace'. + n_ell_bins : int, default 32 + Number of ell bins for pseudo-C_ell analysis. + pol_factor : bool, default True + Apply polarization correction factor in pseudo-C_ell calculations. + nrandom_cell : int, default 10 + Number of random realizations for C_ell error estimation. + cosmo_params : dict, optional + Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + redshift_file : str, optional + Path to n(z) file for theory calculations. Format: columns of (z, n(z)). + + Attributes + ---------- + versions : list of str + Validated catalog versions after processing _leak_corr variants. + cc : dict + Loaded catalog configuration with resolved absolute paths. + catalog_config_path : Path + Resolved path to the catalog configuration file. + treecorr_config : dict + Configuration dictionary passed to TreeCorr correlation objects. + cosmo : pyccl.Cosmology + Cosmology object for theory predictions. + z_dist : ndarray or None + Redshift distribution loaded from redshift_file if provided. + + Notes + ----- + - Path resolution: Relative paths in catalog config are resolved using each + version's 'subdir' field as the base directory. + - Virtual _leak_corr versions: These create deep copies of the base version + config, swapping e1_col/e2_col with e1_col_corrected/e2_col_corrected. + - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, + 'simple' otherwise, following TreeCorr best practices. + """ def __init__( self, versions, From 2a6a3e2c51ffc5f830b0fc5eb8dcde31eea890e7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 18:38:12 +0200 Subject: [PATCH 09/29] refactor: remove redshift_file parameter and use catalog config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eliminate redundant redshift_file parameter and load n(z) directly from catalog configuration. Added get_redshift() method as single source of truth. - Renamed shear.redshift_distr → shear.redshift_path in cat_config.yaml - Added get_redshift(version) method for catalog-aware n(z) loading - Updated calculate_pure_eb(), plot_pure_eb(), calculate_pseudo_cl_eb_cov() to use get_redshift() - Removed redshift_file parameter from __init__, calculate_pure_eb(), plot_pure_eb() signatures and docstrings 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 ++++++++++++++++++ ..._minsep=1_maxsep=250_nbins=20_npatch=1.ini | 124 ++++++++++++++++++ notebooks/cosmo_val/cat_config.yaml | 20 +-- src/sp_validation/cosmo_val.py | 50 +++---- 4 files changed, 279 insertions(+), 39 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 0000000..6be956e --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 0000000..6bc3a7d --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index dd02dfb..cf9e89e 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -211,7 +211,7 @@ SP_test: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_iv e1_col: e1 e1_PSF_col: e1_PSF @@ -883,7 +883,7 @@ SP_v1.4.5: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -928,7 +928,7 @@ SP_v1.4.5.A: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: shapepipe_SPv1.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: g1 e1_PSF_col: e1_PSF @@ -971,7 +971,7 @@ SP_v1.4.5_bright: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_PSF_col: e1_PSF @@ -1014,7 +1014,7 @@ SP_v1.4.5_faint: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_PSF_col: e1_PSF @@ -1057,7 +1057,7 @@ SP_v1.4.5_glass_mock: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 e1_PSF_col: e1_PSF @@ -1100,7 +1100,7 @@ SP_v1.4.5_intermediate: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_PSF_col: e1_PSF @@ -1144,7 +1144,7 @@ SP_v1.4.6: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -1190,7 +1190,7 @@ SP_v1.4.7: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected @@ -1236,7 +1236,7 @@ SP_v1.4.8: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - redshift_distr: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 e1_col_corrected: e1_leak_corrected diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index e93d95b..aa97dac 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -83,8 +83,6 @@ class CosmologyValidation: Number of random realizations for C_ell error estimation. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. - redshift_file : str, optional - Path to n(z) file for theory calculations. Format: columns of (z, n(z)). Attributes ---------- @@ -98,8 +96,6 @@ class CosmologyValidation: Configuration dictionary passed to TreeCorr correlation objects. cosmo : pyccl.Cosmology Cosmology object for theory predictions. - z_dist : ndarray or None - Redshift distribution loaded from redshift_file if provided. Notes ----- @@ -136,7 +132,6 @@ def __init__( pol_factor=True, nrandom_cell=10, cosmo_params=None, - redshift_file=None, ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -167,8 +162,6 @@ def __init__( # Use Planck 2018 defaults self.cosmo = get_cosmo() - # load redshift distribution from file if provided - self.z_dist = np.loadtxt(redshift_file) if redshift_file is not None else None self.treecorr_config = { "ra_units": "degrees", @@ -241,6 +234,24 @@ def resolve_paths_for_version(ver): if not os.path.exists(cc["paths"]["output"]): os.mkdir(cc["paths"]["output"]) + def get_redshift(self, version): + """Load redshift distribution for a catalog version. + + Parameters + ---------- + version : str + Catalog version identifier + + Returns + ------- + z : ndarray + Redshift values + nz : ndarray + n(z) probability density + """ + redshift_path = self.cc[version]["shear"]["redshift_path"] + return np.loadtxt(redshift_path, unpack=True) + def compute_survey_stats( self, ver, @@ -1684,7 +1695,6 @@ def calculate_pure_eb( var_method="jackknife", cov_path_int=None, cosmo_cov=None, - redshift_file=None, n_samples=1000, ): """ @@ -1782,18 +1792,8 @@ def calculate_pure_eb( # Get redshift distribution if using analytic covariance if cov_path_int is not None: - if redshift_file is None: - try: - print("Inheriting redshift distribution from self.z_dist") - z_dist = self.z_dist - except AttributeError: - raise ValueError( - "redshift distribution must be provided either to this function" - " or to the CosmologyValidation class upon creation " - "if using an analytic covariance." - ) - else: - z_dist = np.loadtxt(redshift_file) + z, nz = self.get_redshift(version) + z_dist = np.column_stack([z, nz]) else: z_dist = None @@ -1826,7 +1826,6 @@ def plot_pure_eb( var_method="jackknife", cov_path_int=None, cosmo_cov=None, - redshift_file=None, n_samples=1000, results=None, **kwargs @@ -1864,8 +1863,6 @@ def plot_pure_eb( Path to integration covariance matrix for semi-analytical calculation cosmo_cov : pyccl.Cosmology, optional Cosmology for theoretical predictions in semi-analytical covariance - redshift_file : str, optional - Path to redshift distribution file for semi-analytical covariance n_samples : int Number of Monte Carlo samples for semi-analytical covariance (default: 1000) results : dict or list, optional @@ -1953,7 +1950,6 @@ def plot_pure_eb( var_method=var_method, cov_path_int=cov_path_int, cosmo_cov=cosmo_cov, - redshift_file=redshift_file, n_samples=n_samples, ) @@ -2310,14 +2306,10 @@ def calculate_pseudo_cl_eb_cov(self): self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") lmax = 2 * self.nside - redshift_path = Path(self.cc[ver]["shear"]["redshift_distr"]) - if not redshift_path.is_absolute(): - redshift_path = Path(self.cc[ver]["subdir"]) / redshift_path - path_redshift_distr = str(redshift_path) + z, dndz = self.get_redshift(ver) pw = hp.pixwin(nside, lmax=lmax) # Load redshift distribution and calculate theory C_ell - z, dndz = np.loadtxt(path_redshift_distr, unpack=True) ell = np.arange(1, lmax + 1) fiducial_cl = ( get_theo_c_ell( From 419212bc33cd87e072075df0e1f0e612ebb90908 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 18:55:55 +0200 Subject: [PATCH 10/29] fix: remove obsolete data_base_dir parameter from eb_plots.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update CosmologyValidation call to match current API. The data_base_dir parameter was removed in the catalog config refactor; all paths are now resolved from cat_config.yaml. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py index e0b371c..5d7b740 100644 --- a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py @@ -48,7 +48,6 @@ cv = CosmologyValidation( versions=[config["version"]], - data_base_dir="/n17data/mkilbing/astro/data", ) # %% eb_results = cv.calculate_pure_eb(version=config["version"], **config["pure_eb"]) From a14123cf19cefa6a3dc8b47b41ffa21ce0b304b7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 22:09:41 +0200 Subject: [PATCH 11/29] feat: add glass mock seed variants and v1.4.6 glass mock catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for seed-specific mock catalog variants (e.g., SP_v1.4.5_glass_mock_seed1) by extracting and substituting seed tokens in shear paths. Enables exploring multiple random realizations of the same mock survey. - Add SP_v1.4.6_glass_mock catalog entry with v1.4.6 survey specs - Refactor version processing in __init__ to use recursive ensure_version_exists() - Support _seed variants that deep-copy base config and substitute seed token - Handle _seed_leak_corr combinations by materializing seed config first - Add explicit error checking for missing seed tokens in paths - Add regression tests for seed variant creation and error cases Seed variant examples: - SP_v1.4.5_glass_mock_seed1 → unions_glass_sim_00001_4096.fits - SP_v1.4.6_glass_mock_seed12 → unions_glass_sim_00012_4096.fits - SP_v1.4.5_glass_mock_seed1_leak_corr → combines both transforms 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 44 +++++++++++ src/sp_validation/cosmo_val.py | 95 +++++++++++++++++------ src/sp_validation/tests/test_cosmo_val.py | 91 ++++++++++++++++++++++ 3 files changed, 207 insertions(+), 23 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index cf9e89e..71f145c 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1070,6 +1070,50 @@ SP_v1.4.5_glass_mock: e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.6_glass_mock: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkgreen + getdist_colour: 0.0, 0.7, 0.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits + SP_v1.4.5_intermediate: subdir: /n17data/murray/unions_cats pipeline: SP diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index aa97dac..18dcf71 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,6 +1,7 @@ # %% import copy import os +import re from pathlib import Path import colorama @@ -106,6 +107,34 @@ class CosmologyValidation: - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, 'simple' otherwise, following TreeCorr best practices. """ + + _SEED_SUFFIX_RE = re.compile(r"^(?P.+)_seed(?P\d+)$") + _SEED_TOKEN_RE = re.compile(r"seed(?P[_-]?)(?P\d+)") + + @classmethod + def _split_seed_variant(cls, version): + """Return the base version and seed label if version encodes a seed.""" + match = cls._SEED_SUFFIX_RE.match(version) + if match is None: + return None, None + return match.group("base"), match.group("seed") + + @classmethod + def _apply_seed_token(cls, path, seed_value, version, base_version, catalog_config): + """Replace the final seed token in the path with the requested seed value.""" + matches = list(cls._SEED_TOKEN_RE.finditer(path)) + if not matches: + raise ValueError( + f"Cannot materialize '{version}': shear path '{path}' for base version " + f"'{base_version}' does not contain a 'seed' token. " + f"Update {catalog_config} or drop the seed suffix." + ) + match = matches[-1] + sep = match.group("sep") or "" + replacement = f"seed{sep}{seed_value}" + start, end = match.span() + return f"{path[:start]}{replacement}{path[end:]}" + def __init__( self, versions, @@ -191,38 +220,58 @@ def resolve_paths_for_version(ver): resolve_paths_for_version("nz") processed = {"nz"} final_versions = [] + leak_suffix = "_leak_corr" - for ver in versions: - if ver.endswith("_leak_corr"): - base_ver = ver.replace("_leak_corr", "") - target = base_ver - if base_ver not in cc: - raise KeyError( - f"Base version {base_ver} not found for {ver} in config file " - f"{catalog_config}" - ) - if "e1_col_corrected" not in cc[base_ver]["shear"]: + def ensure_version_exists(ver): + if ver in processed: + return + + if ver in cc: + resolve_paths_for_version(ver) + processed.add(ver) + return + + seed_base, seed_label = self._split_seed_variant(ver) + + if ver.endswith(leak_suffix): + base_ver = ver[: -len(leak_suffix)] + ensure_version_exists(base_ver) + shear_cfg = cc[base_ver]["shear"] + if "e1_col_corrected" not in shear_cfg or "e2_col_corrected" not in shear_cfg: raise ValueError( f"{base_ver} does not have e1_col_corrected/e2_col_corrected " f"fields; cannot create {ver}" ) - else: - target = ver if ver not in cc: - raise KeyError( - f"Version string {ver} not found in config file " - f"{catalog_config}" + cc[ver] = copy.deepcopy(cc[base_ver]) + cc[ver]["shear"]["e1_col"] = shear_cfg["e1_col_corrected"] + cc[ver]["shear"]["e2_col"] = shear_cfg["e2_col_corrected"] + resolve_paths_for_version(ver) + processed.add(ver) + return + + if seed_base is not None: + ensure_version_exists(seed_base) + if ver not in cc: + cc[ver] = copy.deepcopy(cc[seed_base]) + seed_path = self._apply_seed_token( + cc[seed_base]["shear"]["path"], + seed_label, + ver, + seed_base, + catalog_config, ) + cc[ver]["shear"]["path"] = seed_path + resolve_paths_for_version(ver) + processed.add(ver) + return - if target not in processed: - resolve_paths_for_version(target) - processed.add(target) - - if ver.endswith("_leak_corr"): - cc[ver] = copy.deepcopy(cc[base_ver]) - cc[ver]["shear"]["e1_col"] = cc[base_ver]["shear"]["e1_col_corrected"] - cc[ver]["shear"]["e2_col"] = cc[base_ver]["shear"]["e2_col_corrected"] + raise KeyError( + f"Version string {ver} not found in config file {catalog_config}" + ) + for ver in versions: + ensure_version_exists(ver) final_versions.append(ver) self.versions = final_versions diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 55b8f80..16a4c48 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -11,6 +11,7 @@ import os import pytest +import yaml from sp_validation.cosmo_val import CosmologyValidation @@ -42,6 +43,57 @@ def base_config(self, tmp_path): "nbins": 20, } + @staticmethod + def _make_seed_config(tmp_path, shear_filename): + """Create a minimal catalog config for seed variant testing.""" + base_version = "TestCatalog" + base_dir = tmp_path / "catalog" + base_dir.mkdir() + (base_dir / shear_filename).touch() + + star_filename = "star_seed_1234.fits" + (base_dir / star_filename).touch() + + nz_dir = tmp_path / "nz" + nz_dir.mkdir() + (nz_dir / "dndz.txt").write_text("0.1 1.0\n") + + output_dir = tmp_path / "output" + output_dir.mkdir() + + config_path = tmp_path / "seed_config.yaml" + config_data = { + "nz": { + "subdir": str(nz_dir), + "dndz": {"blind": "A", "path": "dndz.txt"}, + }, + "paths": {"output": str(output_dir)}, + base_version: { + "subdir": str(base_dir), + "pipeline": "SP", + "shear": { + "path": shear_filename, + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_col_corrected": "e1_corr", + "e2_col_corrected": "e2_corr", + }, + "star": {"path": star_filename}, + }, + } + config_path.write_text(yaml.dump(config_data, sort_keys=False)) + + params = { + "catalog_config": str(config_path), + "output_dir": str(output_dir), + "npatch": 1, + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + } + return params, base_version + @pytest.mark.parametrize( "version,e1_col,e2_col", [ @@ -125,3 +177,42 @@ def test_additive_bias_leak_corrected_columns(self, base_config, version): # Verify the values are numeric assert isinstance(cv.c1[version_leak_corr], float) assert isinstance(cv.c2[version_leak_corr], float) + + def test_seed_variant_updates_shear_path(self, tmp_path): + """Seeded versions should materialize a seed-specific shear path.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + seed_version = f"{base_version}_seed007" + + cv = CosmologyValidation(versions=[seed_version], **params) + + assert cv.versions == [seed_version] + assert seed_version in cv.cc + assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + + def test_seed_leak_corr_materializes_seed_first(self, tmp_path): + """_seed_leak_corr should clone the seed variant before leak fixes.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + leak_version = f"{base_version}_seed007_leak_corr" + seed_version = f"{base_version}_seed007" + + cv = CosmologyValidation(versions=[leak_version], **params) + + assert cv.versions == [leak_version] + assert seed_version in cv.cc + assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + assert cv.cc[leak_version]["shear"]["e1_col"] == "e1_corr" + assert cv.cc[leak_version]["shear"]["e2_col"] == "e2_corr" + + def test_seed_variant_without_token_errors(self, tmp_path): + """Missing seed token in shear path should raise a descriptive error.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_base.fits" + ) + seed_version = f"{base_version}_seed123" + + with pytest.raises(ValueError, match="seed"): + CosmologyValidation(versions=[seed_version], **params) From 8d14a83d8453542ac83481c490ad534b7e89c202 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 22:14:45 +0200 Subject: [PATCH 12/29] test: add v1.4.6 glass mock to base tests and seed variant test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add SP_v1.4.6_glass_mock to parametrized additive bias tests - Update SP_v1.4.6_glass_mock shear path to glass_mock_v1.4.6 directory - Add test_v1_4_6_glass_mock_seed_variant to verify seed9 variant loads correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 2 +- src/sp_validation/tests/test_cosmo_val.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 71f145c..bcc4274 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1100,7 +1100,7 @@ SP_v1.4.6_glass_mock: shear: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00000_4096.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 16a4c48..0087b6d 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -100,6 +100,7 @@ def _make_seed_config(tmp_path, shear_filename): ("SP_v1.4.5", "e1", "e2"), ("SP_v1.4.6", "e1", "e2"), ("SP_v1.4.5_glass_mock", "e1", "e2"), + ("SP_v1.4.6_glass_mock", "e1", "e2"), ("SP_v1.4.5_bright", "e1", "e2"), ("SP_v1.4.5_faint", "e1", "e2"), ("SP_v1.4.5_intermediate", "e1", "e2"), @@ -216,3 +217,24 @@ def test_seed_variant_without_token_errors(self, tmp_path): with pytest.raises(ValueError, match="seed"): CosmologyValidation(versions=[seed_version], **params) + + def test_v1_4_6_glass_mock_seed_variant(self, base_config): + """Test that v1.4.6 glass mock seed variant loads with correct path.""" + seed = 9 + seed_version = f"SP_v1.4.6_glass_mock_seed{seed}" + + cv = CosmologyValidation( + versions=[seed_version], + **base_config, + ) + + # Verify version was created + assert cv.versions == [seed_version] + assert seed_version in cv.cc + + # Verify seed was substituted in shear path + expected_filename = f"unions_glass_sim_{seed:05d}_4096.fits" + assert expected_filename in cv.cc[seed_version]["shear"]["path"] + + # Verify path points to v1.4.6 glass mock directory + assert "glass_mock_v1.4.6" in cv.cc[seed_version]["shear"]["path"] From 60ffcd3e45d67cde114d9040f79e6d61d146f558 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Oct 2025 23:01:58 +0200 Subject: [PATCH 13/29] refactor: use path templates for elegant seed variant handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace brittle regex-based seed token detection with config-driven path templates using Python string formatting. Each catalog specifies a path_template with {seed:05d} or {seed} placeholders. - Add path_template fields to SP_v1.4.5_glass_mock and SP_v1.4.6_glass_mock - Simplify _split_seed_variant to pure string operations (no regex) - Add _materialize_seed_path using .format() for clean templating - Fallback to legacy seed-token extraction if no template provided - All 15 tests pass including new v1.4.6 glass mock seed9 test Benefits: - Per-catalog control of path formatting without code changes - No complex regex fragility - Clear, self-documenting config entries 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 4 +- src/sp_validation/cosmo_val.py | 86 +++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index bcc4274..219f054 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1057,6 +1057,7 @@ SP_v1.4.5_glass_mock: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + path_template: /n09data/guerrini/glass_mock/results/unions_glass_sim_{seed:05d}_4096.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 @@ -1100,7 +1101,8 @@ SP_v1.4.6_glass_mock: shear: R: 1.0 covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00000_4096.fits + path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00001_4096.fits + path_template: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_{seed:05d}_4096.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w e1_col: e1 diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 18dcf71..07ded12 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,7 +1,6 @@ # %% import copy import os -import re from pathlib import Path import colorama @@ -108,32 +107,73 @@ class CosmologyValidation: 'simple' otherwise, following TreeCorr best practices. """ - _SEED_SUFFIX_RE = re.compile(r"^(?P.+)_seed(?P\d+)$") - _SEED_TOKEN_RE = re.compile(r"seed(?P[_-]?)(?P\d+)") - - @classmethod - def _split_seed_variant(cls, version): + @staticmethod + def _split_seed_variant(version): """Return the base version and seed label if version encodes a seed.""" - match = cls._SEED_SUFFIX_RE.match(version) - if match is None: + if "_seed" not in version: + return None, None + base, seed_label = version.rsplit("_seed", 1) + if not base or not seed_label.isdigit(): return None, None - return match.group("base"), match.group("seed") + return base, seed_label - @classmethod - def _apply_seed_token(cls, path, seed_value, version, base_version, catalog_config): - """Replace the final seed token in the path with the requested seed value.""" - matches = list(cls._SEED_TOKEN_RE.finditer(path)) - if not matches: + @staticmethod + def _materialize_seed_path( + base_cfg, seed_label, version, base_version, catalog_config + ): + """Render the seed-specific shear path using Python string formatting.""" + shear_cfg = base_cfg["shear"] + template = shear_cfg.get("path_template") + + try: + seed_value = int(seed_label) + except ValueError as error: + raise ValueError( + f"Seed suffix for '{version}' is not numeric; cannot materialize path." + ) from error + + format_context = {"seed": seed_value, "seed_label": seed_label} + + if template: + try: + return template.format(**format_context) + except KeyError as error: + raise KeyError( + f"Missing placeholder '{error.args[0]}' in path_template for " + f"'{base_version}' while materializing '{version}'. Update " + f"{catalog_config}." + ) from error + except ValueError as error: + raise ValueError( + f"Invalid format specification in path_template for '{base_version}' " + f"while materializing '{version}'." + ) from error + + path = shear_cfg.get("path", "") + token_start = path.rfind("seed") + if token_start == -1: + raise ValueError( + f"Cannot materialize '{version}': '{base_version}' lacks a shear " + f"path_template and its shear path '{path}' does not contain a 'seed' " + f"token. Update {catalog_config}." + ) + cursor = token_start + 4 # len("seed") + if cursor < len(path) and not path[cursor].isdigit(): + cursor += 1 + digit_start = cursor + while cursor < len(path) and path[cursor].isdigit(): + cursor += 1 + digit_end = cursor + digits = path[digit_start:digit_end] + if not digits: raise ValueError( f"Cannot materialize '{version}': shear path '{path}' for base version " - f"'{base_version}' does not contain a 'seed' token. " - f"Update {catalog_config} or drop the seed suffix." + f"'{base_version}' lacks digits after the seed token. Update " + f"{catalog_config}." ) - match = matches[-1] - sep = match.group("sep") or "" - replacement = f"seed{sep}{seed_value}" - start, end = match.span() - return f"{path[:start]}{replacement}{path[end:]}" + + template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" + return template.format(**format_context) def __init__( self, @@ -254,8 +294,8 @@ def ensure_version_exists(ver): ensure_version_exists(seed_base) if ver not in cc: cc[ver] = copy.deepcopy(cc[seed_base]) - seed_path = self._apply_seed_token( - cc[seed_base]["shear"]["path"], + seed_path = self._materialize_seed_path( + cc[seed_base], seed_label, ver, seed_base, From 8957ce2756622eeada1b5fc335d0e6ce5d370f6f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 17 Oct 2025 13:59:26 +0200 Subject: [PATCH 14/29] test: add v1.4.6 glass mock default seed test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verify that SP_v1.4.6_glass_mock without a seed suffix correctly uses the default seed 00001 from the configured path field. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/sp_validation/tests/test_cosmo_val.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 0087b6d..42759f8 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -238,3 +238,19 @@ def test_v1_4_6_glass_mock_seed_variant(self, base_config): # Verify path points to v1.4.6 glass mock directory assert "glass_mock_v1.4.6" in cv.cc[seed_version]["shear"]["path"] + + def test_v1_4_6_glass_mock_default_seed(self, base_config): + """Test that glass mock without seed suffix uses the default seed_00001.""" + cv = CosmologyValidation( + versions=["SP_v1.4.6_glass_mock"], + **base_config, + ) + + # Verify version loads without seed suffix + assert cv.versions == ["SP_v1.4.6_glass_mock"] + assert "SP_v1.4.6_glass_mock" in cv.cc + + # Verify it uses the default path (seed_00001 for v1.4.6) + path = cv.cc["SP_v1.4.6_glass_mock"]["shear"]["path"] + assert "unions_glass_sim_00001_4096.fits" in path + assert "glass_mock_v1.4.6" in path From bdc14617cc0c66cc35ce67ec81590f328ed2f3f0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 23 Oct 2025 13:53:19 +0200 Subject: [PATCH 15/29] consistent b-mode ylims --- src/sp_validation/b_modes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index d24f9e1..09ce030 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -659,6 +659,7 @@ def plot_pure_eb_correlations( ax.set(xscale="log", xlabel=r"$\theta$ [arcmin]") ax.axhline(0, alpha=0.3, color="k", linestyle="--", linewidth=0.5) ax.legend(loc="upper left") + ax.set_ylim(-0.5, 2) # Save the axis limits after data plotting but before adding gray regions original_xlims = [ax.get_xlim() for ax in axs] @@ -1037,6 +1038,7 @@ def plot_cosebis_modes( plt.legend() plt.xlabel("n (mode)") plt.ylabel("E_n, B_n") + plt.ylim(-0.5e-10, 3e-10) # Add scale cut information to title - use actual scale cut from results scale_info = "" @@ -1092,4 +1094,3 @@ def plot_cosebis_covariance_matrix(results, version, var_method, output_path): ax.set_title(f"{version} COSEBIs E/B {var_method} correlation matrix") plt.savefig(output_path, dpi=300, bbox_inches="tight") - From 9d3dac86b0b95d91b147cab3d2f3fadd0284155c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 24 Oct 2025 17:24:50 +0200 Subject: [PATCH 16/29] refactor: remove non-functional catalog versions and consolidate tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove 14 non-functional catalog versions from cat_config.yaml that have missing or misconfigured file paths: - LF_matched_SP_v1.0, LF_v1.0, LF_v2.0 - SP_matched_LF_v1.0, SP_v1.0_LFmask_4k, SP_v1.0_LFmask_8k - SP_v1.3_LFmask variants (4k, 8k, F2, SN7, SN8, li_2024, no_alpha) - SP_v1.4-P3_LFmask Retain 2 working LFmask versions: SP_v1.4_LFmask_8k and SP_v1.4_LFmask_8k_noalpha Consolidate test_catalog_paths.py functionality into test_cosmo_val.py: - test_catalog_paths_exist() now programmatically discovers all catalog versions - Simplify test_additive_bias_base_columns() to test only SP_v1.4.5 - Update test_additive_bias_leak_corrected_columns() to test SP_v1.4.6_leak_corr - Result: 10x faster test suite (614s → 25s) All tests pass. Remaining: 21 working catalog versions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- notebooks/cosmo_val/cat_config.yaml | 539 ------------------ src/sp_validation/tests/test_catalog_paths.py | 65 --- src/sp_validation/tests/test_cosmo_val.py | 113 +++- 3 files changed, 90 insertions(+), 627 deletions(-) delete mode 100644 src/sp_validation/tests/test_catalog_paths.py diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 219f054..bb791ef 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -44,65 +44,6 @@ DES: e1_col: obs_e1 e2_col: obs_e2 path: psf_y3a1-v29.fits - -LF_matched_SP_v1.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - pipeline: LF - colour: magenta - getdist_colour: 0.0, 0.5, 0.0 - ls: dashdot - marker: ^ - shear: - R: 1.0 - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - path: masked_lensfit_goldshape_2022v1.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: full_starcat_tmp_1_10_seed_1234.fits - -LF_v1.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/Lensfit - pipeline: LF - colour: r - getdist_colour: 1.0, 0.5, 0.0 - ls: solid - marker: '8' - shear: - R: 1.0 - covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt - path: lensfit_goldshape_2022v1.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: full_starcat_tmp_1_10_seed_1234.fits - -LF_v2.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v2.0/Lensfit - pipeline: LF - colour: r - ls: dashed - marker: p - shear: - R: 1.0 - path: lensfit_goldshape_2022v2_psf1.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1_psf - e2_col: e2_psf - path: Pall_hdu1_THELI_psfs1_mosaic.fits - SP_axel_v0.0: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP @@ -147,40 +88,6 @@ SP_axel_v0.0: e1_col: E1_STAR_HSM e2_col: E2_STAR_HSM path: star_cat.fits - -SP_matched_LF_v1.0: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/matched_LF_SP - pipeline: SP - colour: magenta - getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 - ls: dotted - marker: v - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - SP_test: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -223,7 +130,6 @@ SP_test: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.1.fits - SP_v0.1.1: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 pipeline: SP @@ -257,73 +163,6 @@ SP_v0.1.1: e1_col: E1_STAR_HSM e2_col: E2_STAR_HSM path: star_cat.fits - -SP_v1.0_LFmask_4k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: blue - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dashdot - marker: h - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_2022_v1.0_mtheli4k.fits - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - -SP_v1.0_LFmask_8k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: blue - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - SP_v1.3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -361,324 +200,6 @@ SP_v1.3: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.0.3.fits - -SP_v1.3_LFmask_4k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: green - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dashdot - marker: h - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_2022_v1.3_mtheli4k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - -SP_v1.3_LFmask_8k: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: green - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e1_PSF_col: e1_PSF - e2_col: e2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - -SP_v1.3_LFmask_8k_F2: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: lawngreen - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 9.4 - n_psf: 0.48 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - -SP_v1.3_LFmask_8k_SN7: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: purple - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 10.989 - n_psf: 0.48 - sigma_e: 0.3796 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - -SP_v1.3_LFmask_8k_SN8: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: darkgreen - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 9.7 - n_psf: 0.48 - sigma_e: 0.35 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - -SP_v1.3_LFmask_8k_li_2024: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: deepskyblue - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits - w_col: w - e1_col: e1_cor_2 - e1_PSF_col: e1_PSF - e2_col: e2_cor_2 - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - -SP_v1.3_LFmask_8k_no_alpha: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask - pipeline: SP - colour: blueviolet - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dotted - marker: h - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: false - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM - hdu: 1 - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - label: SP_LFmask_psf - shear: - R: 1.0 - R11: 1.0 - R22: 1.0 - path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits - w_col: w - e1_col: e1_cor - e1_PSF_col: e1_PSF - e2_col: e2_cor - e2_PSF_col: e2_PSF - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - SP_v1.4-P3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 pipeline: SP @@ -724,50 +245,6 @@ SP_v1.4-P3: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_P3_v1.4.fits - -SP_v1.4-P3_LFmask: - subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask/P3 - pipeline: SP - colour: black - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - ls: dashdot - marker: h - cov_th: - A: 223 - n_e: 7.44 - n_psf: 0.32 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P3_LFmask.fits - path_tau: tau_stats_SP_v1.4-P3_LFmask.fits - path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits - path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits - psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM - hdu: 1 - path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM - shear: - R: 1.0 - path: unions_shapepipe_2022_v1.4_mtheli4k.fits - w_col: w - e1_col: e1 - e2_col: e2 - star: - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits - SP_v1.4.1_noleakage: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe pipeline: SP @@ -809,7 +286,6 @@ SP_v1.4.1_noleakage: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.1.fits - SP_v1.4.2: subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 pipeline: SP @@ -851,7 +327,6 @@ SP_v1.4.2: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -897,7 +372,6 @@ SP_v1.4.5: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5.A: subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 pipeline: SP @@ -940,7 +414,6 @@ SP_v1.4.5.A: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_bright: subdir: /n17data/murray/unions_cats pipeline: SP @@ -983,7 +456,6 @@ SP_v1.4.5_bright: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_faint: subdir: /n17data/murray/unions_cats pipeline: SP @@ -1026,7 +498,6 @@ SP_v1.4.5_faint: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_glass_mock: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1070,7 +541,6 @@ SP_v1.4.5_glass_mock: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.6_glass_mock: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1115,7 +585,6 @@ SP_v1.4.6_glass_mock: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.5_intermediate: subdir: /n17data/murray/unions_cats pipeline: SP @@ -1158,7 +627,6 @@ SP_v1.4.5_intermediate: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.6: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1204,7 +672,6 @@ SP_v1.4.6: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.7: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1250,7 +717,6 @@ SP_v1.4.7: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4.8: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1296,7 +762,6 @@ SP_v1.4.8: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits - SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP @@ -1343,7 +808,6 @@ SP_v1.4_LFmask_8k: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - SP_v1.4_LFmask_8k_noalpha: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP @@ -1390,7 +854,6 @@ SP_v1.4_LFmask_8k_noalpha: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - SP_v1.5.4: subdir: /n17data/UNIONS/WL/v1.5.x pipeline: SP @@ -1432,12 +895,10 @@ SP_v1.5.4: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.5.a.fits - nz: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz dndz: blind: A path: dndz - paths: output: ./output diff --git a/src/sp_validation/tests/test_catalog_paths.py b/src/sp_validation/tests/test_catalog_paths.py deleted file mode 100644 index 51e4322..0000000 --- a/src/sp_validation/tests/test_catalog_paths.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Lightweight checks that catalog paths referenced in ``cat_config.yaml`` exist. - -These tests do not load the data; they only ensure that the configuration points -to readable files so that downstream validation runs fail fast if a path drifts. -""" - -from __future__ import annotations - -from collections import defaultdict -from pathlib import Path -from typing import Dict, Iterator, Tuple - -import yaml - - -CATALOG_CONFIG = ( - Path(__file__).resolve().parents[3] / "notebooks" / "cosmo_val" / "cat_config.yaml" -) - - -def _resolve(base: Path, candidate: str) -> Path: - """Return an absolute path given a base directory and a candidate string.""" - candidate_path = Path(candidate) - return candidate_path if candidate_path.is_absolute() else base / candidate_path - - -def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: - """Yield (name, entry) pairs for catalog-like entries in the config.""" - for name, entry in config.items(): - if not isinstance(entry, dict): - continue - if "subdir" not in entry: - continue - yield name, entry - - -def test_catalog_files_exist(): - """Ensure every shear/star/psf entry references an on-disk file.""" - config = yaml.safe_load(CATALOG_CONFIG.read_text()) - - missing = defaultdict(set) - - for name, entry in _iter_catalog_entries(config): - base = Path(entry["subdir"]) - assert base.is_absolute(), f"{name}: subdir must be absolute ({base})" - - if name == "nz": - dndz_path = _resolve(base, entry["dndz"]["path"]) - assert ( - dndz_path.parent.is_dir() - ), f"{name}: dndz parent directory missing ({dndz_path.parent})" - continue - - for block_name in ("shear", "star", "psf"): - block = entry.get(block_name) - if not block: - continue - resolved_path = _resolve(base, block["path"]) - if not resolved_path.is_file(): - missing[name].add(block_name) - - assert not missing, ( - "Catalog configuration references missing files: " - f"{ {name: sorted(blocks) for name, blocks in missing.items()} }" - ) diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 42759f8..afcd310 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -9,6 +9,9 @@ """ import os +from collections import defaultdict +from pathlib import Path +from typing import Dict, Iterator, Tuple import pytest import yaml @@ -94,28 +97,17 @@ def _make_seed_config(tmp_path, shear_filename): } return params, base_version - @pytest.mark.parametrize( - "version,e1_col,e2_col", - [ - ("SP_v1.4.5", "e1", "e2"), - ("SP_v1.4.6", "e1", "e2"), - ("SP_v1.4.5_glass_mock", "e1", "e2"), - ("SP_v1.4.6_glass_mock", "e1", "e2"), - ("SP_v1.4.5_bright", "e1", "e2"), - ("SP_v1.4.5_faint", "e1", "e2"), - ("SP_v1.4.5_intermediate", "e1", "e2"), - ("SP_v1.4.5.A", "g1", "g2"), - ("SP_v1.4.7", "e1", "e2"), - ("SP_v1.4.8", "e1", "e2"), - ], - ) - def test_additive_bias_base_columns(self, base_config, version, e1_col, e2_col): + def test_additive_bias_base_columns(self, base_config): """Test additive bias calculation using base ellipticity columns. This test initializes CosmologyValidation without an ellipticity_suffix, which means it will use the default columns defined in the catalog - configuration. + configuration. Tests SP_v1.4.5 with full additive bias computation. """ + version = "SP_v1.4.5" + e1_col = "e1" + e2_col = "e2" + cv = CosmologyValidation( versions=[version], **base_config, @@ -140,16 +132,16 @@ def test_additive_bias_base_columns(self, base_config, version, e1_col, e2_col): assert isinstance(cv.c1[version], float) assert isinstance(cv.c2[version], float) - @pytest.mark.parametrize("version", ["SP_v1.4.5"]) - def test_additive_bias_leak_corrected_columns(self, base_config, version): + def test_additive_bias_leak_corrected_columns(self, base_config): """Test additive bias calculation using leak-corrected columns. - This test requests a leak-corrected version by passing "{version}_leak_corr" + This test requests a leak-corrected version by passing "SP_v1.4.6_leak_corr" as the version name. The CosmologyValidation class automatically detects the _leak_corr suffix and creates the config entry using e1_col_corrected and e2_col_corrected from the base version. """ - version_leak_corr = f"{version}_leak_corr" + base_version = "SP_v1.4.6" + version_leak_corr = f"{base_version}_leak_corr" cv = CosmologyValidation( versions=[version_leak_corr], @@ -164,8 +156,8 @@ def test_additive_bias_leak_corrected_columns(self, base_config, version): assert cv.cc[version_leak_corr]["shear"]["e2_col"] == "e2_leak_corrected" # Verify original config entry remains unchanged - assert cv.cc[version]["shear"]["e1_col"] == "e1" - assert cv.cc[version]["shear"]["e2_col"] == "e2" + assert cv.cc[base_version]["shear"]["e1_col"] == "e1" + assert cv.cc[base_version]["shear"]["e2_col"] == "e2" # Calculate additive bias cv.calculate_additive_bias() @@ -179,6 +171,81 @@ def test_additive_bias_leak_corrected_columns(self, base_config, version): assert isinstance(cv.c1[version_leak_corr], float) assert isinstance(cv.c2[version_leak_corr], float) + @staticmethod + def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: + """Yield (name, entry) pairs for catalog-like entries in the config.""" + for name, entry in config.items(): + if not isinstance(entry, dict): + continue + if "subdir" not in entry: + continue + yield name, entry + + @staticmethod + def _resolve(base: Path, candidate: str) -> Path: + """Return an absolute path given a base directory and a candidate string.""" + candidate_path = Path(candidate) + return candidate_path if candidate_path.is_absolute() else base / candidate_path + + def test_catalog_paths_exist(self, base_config): + """Verify that catalog paths for active versions exist on disk. + + This is a lightweight test that checks that all files referenced in the + catalog configuration for UNIONS analysis versions actually exist. It + discovers versions programmatically from cat_config.yaml rather than + using hardcoded lists. + """ + # Get the path to catalog config + repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + ) + catalog_config_path = os.path.join( + repo_root, "notebooks", "cosmo_val", "cat_config.yaml" + ) + + config = yaml.safe_load(Path(catalog_config_path).read_text()) + + missing = defaultdict(set) + working = [] + nonfunctional = defaultdict(set) + + for version, entry in self._iter_catalog_entries(config): + # Skip nz entries and versions already tested in heavy tests + if version == "nz": + continue + + base = Path(entry["subdir"]) + version_missing = set() + + # Check shear, star, and psf files + for block_name in ("shear", "star", "psf"): + block = entry.get(block_name) + if not block: + continue + resolved_path = self._resolve(base, block["path"]) + if not resolved_path.is_file(): + version_missing.add(block_name) + + if version_missing: + nonfunctional[version] = version_missing + else: + working.append(version) + + # Print summary + print(f"\n✓ Working versions ({len(working)}):") + for v in sorted(working): + print(f" - {v}") + + if nonfunctional: + print(f"\n✗ Non-functional versions ({len(nonfunctional)}):") + for v in sorted(nonfunctional.keys()): + print(f" - {v}: missing {nonfunctional[v]}") + + assert not nonfunctional, ( + "Catalog configuration references missing files: " + f"{dict(nonfunctional)}" + ) + def test_seed_variant_updates_shear_path(self, tmp_path): """Seeded versions should materialize a seed-specific shear path.""" params, base_version = self._make_seed_config( From 27c0cbf26256a27fd3500353cb01d1bf40a0a553 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 27 Oct 2025 18:52:45 +0100 Subject: [PATCH 17/29] rework rho_tau.py to use consistent CosmologyValidation basename --- src/sp_validation/cosmo_val.py | 45 ++++- src/sp_validation/rho_tau.py | 357 ++++++++++++++++++++------------- 2 files changed, 260 insertions(+), 142 deletions(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index b00287f..d3a46ea 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -21,7 +21,11 @@ from .cosmology import get_cosmo, get_theo_c_ell from .plots import FootprintPlotter -from .rho_tau import get_params_rho_tau, get_rho_tau_w_cov, get_samples +from .rho_tau import ( + get_params_rho_tau, + get_rho_tau_w_cov, + get_samples, +) # %% @@ -593,6 +597,17 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise + def basename(self, version, treecorr_config=None, npatch=None): + cfg = treecorr_config or self.treecorr_config + patches = npatch or self.npatch + return ( + f"{version}_minsep={cfg['min_sep']}" + f"_maxsep={cfg['max_sep']}" + f"_nbins={cfg['nbins']}" + f"_npatch={patches}" + ) + + def calculate_rho_tau_stats(self): out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): @@ -600,13 +615,16 @@ def calculate_rho_tau_stats(self): self.print_start("Rho stats") for ver in self.versions: + base = self.basename(ver) rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( self.cc, ver, self.treecorr_config, out_dir, + base, method=self.cov_estimate_method, cov_rho=self.compute_cov_rho, + npatch=self.npatch, ) self.print_done("Rho stats finished") @@ -726,7 +744,9 @@ def calculate_ellipticity_dispersion(self): self._ellipticity_dispersion = ellipticity_dispersion def plot_rho_stats(self, abs=False): - filenames = [f"rho_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"rho_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "rho_stats.png" self.rho_stat_handler.plot_rho_stats( @@ -746,7 +766,9 @@ def plot_rho_stats(self, abs=False): ) def plot_tau_stats(self, plot_tau_m=False): - filenames = [f"tau_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"tau_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "tau_stats.png" self.tau_stat_handler.plot_tau_stats( @@ -826,9 +848,12 @@ def calculate_rho_tau_fits(self): self.cov_estimate_method, None ) + base = self.basename(ver) + flat_samples, result, q = get_samples( self.psf_fitter, ver, + base, cov_type=self.cov_estimate_method, apply_debias=npatch, sampler=self.rho_tau_method, @@ -838,7 +863,7 @@ def calculate_rho_tau_fits(self): self.rho_tau_fits["result_list"].append(result) self.rho_tau_fits["q_list"].append(q) - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] xi_psf_sys_samples = np.array([]).reshape(0, nbins) @@ -883,7 +908,7 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for i in range(100): self.psf_fitter.plot_xi_psf_sys( flat_sample[-i + 1], ver, color, alpha=0.1 @@ -933,7 +958,7 @@ def plot_rho_tau_fits(self): self.versions, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for yscale in ("linear", "log"): out_path = os.path.abspath( f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" @@ -2582,10 +2607,16 @@ def calculate_pseudo_cl_eb_cov(self): lmax = 2 * self.nside z, dndz = self.get_redshift(ver) + ell = np.arange(1, lmax + 1) pw = hp.pixwin(nside, lmax=lmax) + if pw.shape[0] != len(ell) + 1: + raise ValueError( + "Unexpected pixwin length for lmax=" + f"{lmax}: got {pw.shape[0]}, expected {len(ell)+1}" + ) + pw = pw[1:len(ell)+1] # Load redshift distribution and calculate theory C_ell - ell = np.arange(1, lmax + 1) fiducial_cl = ( get_theo_c_ell( ell=ell, diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 29ac84a..2647b37 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -1,5 +1,6 @@ import os import time +from pathlib import Path import matplotlib.scale as mscale import matplotlib.ticker as ticker @@ -69,6 +70,9 @@ def get_transform(self): ] +def _extract_xip(correlations): + """Return flattened array of xip values from a list of correlations.""" + return np.array([corr.xip for corr in correlations]).flatten() def get_params_rho_tau(cat, survey="other"): # Set parameters @@ -111,42 +115,59 @@ def get_params_rho_tau(cat, survey="other"): return params -def get_rho_tau_w_cov(config, version, treecorr_config, outdir, method, cov_rho=False): - """ - Method to compute the covariance matrices of rho and tau-statistics of a given list of versions in cosmo_val. - Also computes rho and tau-statistics. - - Parameters - ---------- - versions : list - List of versions to compute the covariance matrices for. - method : str - Method to compute the covariance matrices. Options are 'jk' or 'th'. - """ +def get_rho_tau_w_cov( + config, + version, + treecorr_config, + outdir, + base, + method, + cov_rho=False, + npatch=None, +): + """Compute rho/tau statistics and, if requested, their covariance.""" if method == "th": nbin_ang, nbin_rad = 100, 200 rho_stat_handler, tau_stat_handler = get_rho_tau( - config, version, treecorr_config, outdir, cov_rho=cov_rho + config, + version, + treecorr_config, + outdir, + base, + cov_rho=cov_rho, ) get_theory_cov( config, version, treecorr_config, outdir, + base, nbin_ang=nbin_ang, nbin_rad=nbin_rad, ) return rho_stat_handler, tau_stat_handler elif method == "jk": - return get_jackknife_cov(config, version, treecorr_config, outdir) + return get_jackknife_cov( + config, + version, + treecorr_config, + outdir, + base, + npatch=npatch, + ) elif method == "sim": - if os.path.exists(outdir + "/cov_tau_" + version + "_th.npy"): - print( - f"Covariance from simulation available at the following file: {outdir+'/cov_tau_'+version+'_th.npy'}" - ) - print(f"Computing rho and tau statistics for the version: {version}") + tau_cov_path = Path(outdir) / f"cov_tau_{base}_th.npy" + + if tau_cov_path.exists(): + print(f"Found existing covariance at {tau_cov_path}") + print(f"Computing rho/tau statistics for {version}") return get_rho_tau( - config, version, treecorr_config, outdir, cov_rho=cov_rho + config, + version, + treecorr_config, + outdir, + base, + cov_rho=cov_rho, ) else: raise ValueError( @@ -156,7 +177,15 @@ def get_rho_tau_w_cov(config, version, treecorr_config, outdir, method, cov_rho= raise ValueError("Method must be either 'jk' or 'th' or 'sim'.") -def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): +def get_rho_tau( + config, + version, + treecorr_config, + outdir, + base, + cov_rho=False, + npatch=None, +): """ Compute rho and tau statistics for a given version of the catalogue. @@ -167,7 +196,7 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): version : str Version of the catalogue to use. treecorr_config : dict - Configuraion for treecorr. + TreeCorr configuration (must include 'min_sep', 'max_sep', and 'nbins'). outdir : str Output directory. """ @@ -177,16 +206,15 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): print("Compute Rho and Tau statistics for the version: ", version) start_time = time.time() - out_base = f"rho_stats_{version}.fits" - out_path = f"{outdir}/{out_base}" + rho_path = Path(outdir) / f"rho_stats_{base}.fits" rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=True ) - if os.path.exists(out_path): - print(f"Skipping rho statistics computation, file {out_path} already exists.") - rho_stat_handler.load_rho_stats(out_base) + if rho_path.exists(): + print(f"Skipping rho statistics computation, file {rho_path} already exists.") + rho_stat_handler.load_rho_stats(rho_path.name) else: rho_stat_handler.catalogs.set_params(params, outdir) @@ -208,20 +236,19 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): ) if cov_rho: - if not os.path.exists(outdir + "/cov_rho_" + version + ".npy"): - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() + cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + if not cov_rho_path.exists(): rho_stat_handler.compute_rho_stats( version, - out_base, + rho_path.name, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) else: - rho_stat_handler.compute_rho_stats(version, out_base, var_method=None) + rho_stat_handler.compute_rho_stats(version, rho_path.name, var_method=None) - out_base = f"tau_stats_{version}.fits" - out_path = f"{outdir}/{out_base}" + tau_path = Path(outdir) / f"tau_stats_{base}.fits" tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, @@ -230,9 +257,9 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): verbose=True, ) - if os.path.exists(out_path): - print(f"Skipping tau statistics computation, file {out_path} already exists.") - tau_stat_handler.load_tau_stats(out_base) + if tau_path.exists(): + print(f"Skipping tau statistics computation, file {tau_path} already exists.") + tau_stat_handler.load_tau_stats(tau_path.name) else: tau_stat_handler.catalogs.set_params(params, outdir) @@ -266,14 +293,20 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): ) # function to extract the tau_+ - tau_stat_handler.compute_tau_stats(version, out_base, var_method=None) + tau_stat_handler.compute_tau_stats(version, tau_path.name, var_method=None) print(f"Time to compute rho and tau statistics: {time.time() - start_time:.2f} s") return rho_stat_handler, tau_stat_handler def get_theory_cov( - config, version, treecorr_config, outdir, nbin_ang=100, nbin_rad=100 + config, + version, + treecorr_config, + outdir, + base, + nbin_ang=100, + nbin_rad=100, ): """ Compute an analytical estimate of the covariance matrix of rho and tau-statistics. @@ -290,15 +323,17 @@ def get_theory_cov( path_psf = info["psf"]["path"] hdu_psf = info["psf"]["hdu"] - print("Computing the covariance matrix for the version: ", version) - start_time = time.time() + target_cov = Path(outdir) / f"cov_tau_{base}_th.npy" - if os.path.exists(outdir + "/cov_tau_" + version + "_th.npy"): + if target_cov.exists(): print( - f"Skipping covariance computation, file {outdir+'/cov_tau_'+version+'_th.npy'} already exists." + f"Skipping covariance computation, file {target_cov} already exists." ) return + print("Computing the covariance matrix for the version: ", version) + start_time = time.time() + cov_tau_th = CovTauTh( path_gal=path_gal, path_psf=path_psf, @@ -310,28 +345,39 @@ def get_theory_cov( params=params, ) - print( - "--- Computation of the rho and tau statistics for the covariance %s seconds ---" - % (time.time() - start_time) - ) + elapsed = time.time() - start_time + print(f"--- Rho/tau statistics for covariance computed in {elapsed:.2f}s ---") cov = cov_tau_th.build_cov(nbin_ang=nbin_ang, nbin_rad=nbin_rad) - print("--- Covariance computation %s seconds ---" % (time.time() - start_time)) - np.save(outdir + "/cov_tau_" + version + "_th.npy", cov) + print(f"--- Covariance matrix assembled in {time.time() - start_time:.2f}s ---") + target_cov.parent.mkdir(parents=True, exist_ok=True) + np.save(target_cov, cov) print("Saved covariance matrix of version: ", version) del cov_tau_th return -def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): +def get_jackknife_cov( + config, + version, + treecorr_config, + outdir, + base, + npatch, + ncov=100, +): """ Compute the covariance matrix of rho and tau-statistics using the jackknife method. Also compute rho and tau-statistics. """ - if os.path.exists(outdir + "/cov_tau_" + version + "_jk.npy"): + rho_filename = f"rho_stats_{base}.fits" + tau_filename = f"tau_stats_{base}.fits" + tau_cov_path = Path(outdir) / f"cov_tau_{base}_jk.npy" + + if tau_cov_path.exists(): print( - f"Skipping covariance computation, file {outdir+'/cov_tau_'+version+'_jk.npy'} already exists." + f"Skipping covariance computation, file {tau_cov_path} already exists." ) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False @@ -344,19 +390,22 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): verbose=True, ) + rho_path = Path(outdir) / rho_filename + tau_path = Path(outdir) / tau_filename + if rho_path.exists(): + rho_stat_handler.load_rho_stats(rho_path.name) + if tau_path.exists(): + tau_stat_handler.load_tau_stats(tau_path.name) return rho_stat_handler, tau_stat_handler - + params = get_params_rho_tau(config[version], survey=version) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False ) - out_base = f"rho_stats_{version}.fits" - rho_stat_handler.catalogs.set_params(params, outdir) - mask = version != "DES" square_size = params["square_size"] tau_stat_handler = TauStat( @@ -366,25 +415,18 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): verbose=True, ) - out_base = f"tau_stats_{version}.fits" - tau_stat_handler.catalogs.set_params(params, outdir) for i in range(ncov): - if not ( - os.path.exists(outdir + "/cov_tau_" + version + str(i) + ".npy") - and os.path.exists(outdir + "/cov_rho_" + version + str(i) + ".npy") - ): - + tau_chunk = outdir + f"/cov_tau_{version}{i}.npy" + rho_chunk = outdir + f"/cov_rho_{version}{i}.npy" + if not (os.path.exists(tau_chunk) and os.path.exists(rho_chunk)): print( - f"Computing rho-statistics of version {version} for jackknife patch {i+1}/{ncov}" + f"Computing rho-statistics for {version} (patch {i+1}/{ncov})" ) - if ( - f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict.keys() - ): - + if f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict: # Build catalogues rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], @@ -394,7 +436,9 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): hdu=config[version]["psf"]["hdu"], ) - tau_stat_handler.catalogs.catalogs_dict = rho_stat_handler.catalogs.catalogs_dict + tau_stat_handler.catalogs.catalogs_dict = ( + rho_stat_handler.catalogs.catalogs_dict + ) # Build the catalog of galaxies. PSF was computed above tau_stat_handler.build_cat_to_compute_tau( @@ -411,45 +455,48 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): ) npatch = rho_stat_handler.catalogs._params["patch_number"] - field = rho_stat_handler.catalogs.catalogs_dict[f"psf_{version}{i}"].getNField(max_top=int.bit_length(npatch)-1, coords='spherical') + field = rho_stat_handler.catalogs.catalogs_dict[ + f"psf_{version}{i}" + ].getNField(max_top=int.bit_length(npatch) - 1, coords="spherical") patch, centers = field.run_kmeans(npatch) #Update the patch centers of the catalogs for key, cat in rho_stat_handler.catalogs.catalogs_dict.items(): cat._centers = centers - field = cat.getNField(max_top=int.bit_length(npatch)-1, coords='spherical') + field = cat.getNField( + max_top=int.bit_length(npatch) - 1, coords="spherical" + ) cat._patch = field.kmeans_assign_patches(centers) - - - out_base = f"rho_stats_{version}.fits" # Compute and save rho stats - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() rho_stat_handler.compute_rho_stats( version + str(i), - out_base, + rho_filename, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) - out_base = f"tau_stats_{version}.fits" - # function to extract the tau_+ - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() tau_stat_handler.compute_tau_stats( version + str(i), - out_base, + tau_filename, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) #Update the keys in the dictionaries - rho_stat_handler.catalogs.catalogs_dict[f"psf_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_{version}{i}") - rho_stat_handler.catalogs.catalogs_dict[f"psf_error_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_error_{version}{i}") - rho_stat_handler.catalogs.catalogs_dict[f"psf_size_error_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_size_error_{version}{i}") - tau_stat_handler.catalogs.catalogs_dict[f"gal_{version}{i+1}"] = tau_stat_handler.catalogs.catalogs_dict.pop(f"gal_{version}{i}") + rho_dict = rho_stat_handler.catalogs.catalogs_dict + tau_dict = tau_stat_handler.catalogs.catalogs_dict + rho_dict[f"psf_{version}{i+1}"] = rho_dict.pop(f"psf_{version}{i}") + rho_dict[f"psf_error_{version}{i+1}"] = rho_dict.pop( + f"psf_error_{version}{i}" + ) + rho_dict[f"psf_size_error_{version}{i+1}"] = rho_dict.pop( + f"psf_size_error_{version}{i}" + ) + tau_dict[f"gal_{version}{i+1}"] = tau_dict.pop(f"gal_{version}{i}") cov_tau_loc = np.zeros_like(np.load(outdir + f"/cov_tau_{version}0.npy")) cov_rho_loc = np.zeros_like(np.load(outdir + f"/cov_rho_{version}0.npy")) @@ -462,122 +509,162 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): cov_tau = cov_tau_loc / ncov cov_rho = cov_rho_loc / ncov - np.save(outdir + "/cov_tau_" + version + "_jk.npy", cov_tau) - np.save(outdir + "/cov_rho_" + version + ".npy", cov_rho) + tau_cov_path.parent.mkdir(parents=True, exist_ok=True) + np.save(tau_cov_path, cov_tau) + + cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + cov_rho_path.parent.mkdir(parents=True, exist_ok=True) + np.save(cov_rho_path, cov_rho) return rho_stat_handler, tau_stat_handler -def get_samples(psf_fitter, version, cov_type="jk", apply_debias=None, sampler="emcee"): - """ - Samples (alpha, beta, eta) using the sampler 'emcee' or 'lsq' +def get_samples( + psf_fitter, + version, + base, + cov_type="jk", + apply_debias=None, + sampler="emcee", +): + """Return (alpha, beta, eta) samples using ``emcee`` or least squares. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance that provides ``load_*`` helpers. version : str - Version of the catalogue to use. - cov_type : str - Type of covariance matrix to use. Options are 'jk' or 'th' or 'sim'. - apply_debias : int - If not None, apply debiasing to the least square method. - sampler : str - Sampler to use. Options are 'emcee' or 'lsq'. (Default: 'emcee') + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename (e.g. ``SP_v1.4_minsep=…``) used for filenames. + cov_type : str, optional + Covariance label (``'jk'``, ``'th'``, or ``'sim'``). Defaults to ``'jk'``. + apply_debias : int or None, optional + Jackknife patch count used to debias samples. Disabled when ``None``. + sampler : str, optional + ``'emcee'`` for MCMC sampling, ``'lsq'`` for least squares + (default ``'emcee'``). """ if sampler == "emcee": return get_samples_emcee( - psf_fitter, version, cov_type=cov_type, apply_debias=apply_debias + psf_fitter, + version, + base, + cov_type=cov_type, + apply_debias=apply_debias, ) elif sampler == "lsq": return get_samples_lsq( - psf_fitter, version, cov_type=cov_type, apply_debias=apply_debias + psf_fitter, + version, + base, + cov_type=cov_type, + apply_debias=apply_debias, ) else: raise ValueError("Sampler must be either 'emcee' or 'lsq'.") def get_samples_emcee( - psf_fitter, version, nwalkers=124, nsamples=10000, cov_type="jk", apply_debias=None + psf_fitter, + version, + base, + nwalkers=124, + nsamples=10000, + cov_type="jk", + apply_debias=None, ): - """ - Samples (alpha, beta, eta) using the covariance of the tau statistics and emcee. + """Draw (alpha, beta, eta) samples using ``emcee`` and the tau covariance. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance managing rho/tau statistics and covariances. version : str - Version of the catalogue to use. - nwalkers : int - Number of walkers to use in the MCMC. (Default: 124) - nsamples : int - Number of samples to draw from the MCMC. (Default 10000) - cov_type : str - Type of covariance matrix to use. Options are 'jk' or 'th' or 'sim'. (Default: 'jk') + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename for locating statistics/covariance files. + nwalkers : int, optional + Number of walkers for the MCMC run (default ``124``). + nsamples : int, optional + Number of samples drawn per walker (default ``10000``). + cov_type : str, optional + Covariance label (``'jk'``/``'th'``/``'sim'``). Defaults to ``'jk'``. + apply_debias : int or None, optional + Jackknife patch count applied during debiasing. Disabled when ``None``. """ # Load rho and tau stats - psf_fitter.load_rho_stat("rho_stats_" + version + ".fits") - psf_fitter.load_tau_stat("tau_stats_" + version + ".fits") + psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") # Check if the path exists sample_file_path = psf_fitter.get_sample_file_path(version) if os.path.exists(sample_file_path): - print(f"Skipping sample computation, file {sample_file_path} already exists.") + print(f"Skipping sampling; {sample_file_path} exists.") flat_samples = psf_fitter.load_samples(version) mcmc_result, q = psf_fitter.get_mcmc_from_samples(version) print(mcmc_result) # Or run MCMC else: print("MCMC sampling") - psf_fitter.load_covariance("cov_tau_" + version + "_" + cov_type + ".npy") + cov_filename = f"cov_tau_{base}_{cov_type}.npy" + psf_fitter.load_covariance(cov_filename, cov_type="tau") - npatch = apply_debias if (apply_debias is not None) else None + debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.run_chain( nwalkers=nwalkers, nsamples=nsamples, - npatch=npatch, - apply_debias=npatch is not None, + npatch=debias_npatch, + apply_debias=debias_npatch is not None, savefig="mcmc_samples_" + version + ".png", ) psf_fitter.save_samples(flat_samples, version) return flat_samples, mcmc_result, q -def get_samples_lsq(psf_fitter, version, apply_debias=None, cov_type="jk"): - """ - Samples (alpha, beta, eta) using the covariance of the tau statistics and least square method. +def get_samples_lsq( + psf_fitter, + version, + base, + apply_debias=None, + cov_type="jk", +): + """Compute least-squares samples of (alpha, beta, eta) using the tau covariance. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance managing rho/tau statistics and covariances. version : str - Version of the catalogue to use. - apply_debias : int - If not None, apply debiasing to the least square method. (Default: None) + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename for locating statistics/covariance files. + apply_debias : int or None, optional + Jackknife patch count applied during debiasing. Disabled when ``None``. + cov_type : str, optional + Covariance label (defaults to ``'jk'``). """ # Load rho and tau stats - psf_fitter.load_rho_stat("rho_stats_" + version + ".fits") - psf_fitter.load_tau_stat("tau_stats_" + version + ".fits") + psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") # Check if the path exists sample_file_path = psf_fitter.get_sample_path(version) if os.path.exists(sample_file_path): - print(f"Skipping sample computation, file {sample_file_path} already exists.") + print(f"Skipping sampling; {sample_file_path} exists.") flat_samples = psf_fitter.load_samples(version) mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC else: print("Least square sampling") - psf_fitter.load_covariance( - "cov_tau_" + version + "_" + cov_type + ".npy", cov_type="tau" - ) - psf_fitter.load_covariance("cov_rho_" + version + ".npy", cov_type="rho") - npatch = apply_debias if (apply_debias is not None) else None + tau_covariance = f"cov_tau_{base}_{cov_type}.npy" + rho_covariance = f"cov_rho_{base}_jk.npy" + psf_fitter.load_covariance(tau_covariance, cov_type="tau") + psf_fitter.load_covariance(rho_covariance, cov_type="rho") + debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.get_least_squares_params_samples( - npatch=npatch, apply_debias=(npatch is not None) + npatch=debias_npatch, apply_debias=(debias_npatch is not None) ) psf_fitter.save_samples(flat_samples, version) return flat_samples, mcmc_result, q From a1bd87224e5505f0e6a3d4fdb009d5c68ca90ffd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 27 Oct 2025 23:38:33 +0100 Subject: [PATCH 18/29] merge pipeline.sh and cosmosis_fitting, make work with mocks --- cosmo_inference/scripts/cosmosis_fitting.py | 813 ++++++++++++-------- 1 file changed, 478 insertions(+), 335 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 7f4d918..6c2fc8f 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -1,142 +1,215 @@ #!/usr/bin/env python # coding: utf-8 +import argparse import os +import re import sys -from pathlib import Path -import matplotlib.pylab as plt import numpy as np from astropy.io import fits +# ============================================================================ +# SECTION 1: FITS FORMAT CONVERTERS (data → FITS HDU) +# ============================================================================ -# transforms treecorr fits file of correlation functions into CosmoSIS-friendly 2pt FITS extension to be read by 2pt_likelihood -def treecorr_to_fits(filename1, filename2): +def _create_2pt_hdu(values, theta, name, quant1, quant2): + """Create standardized 2-point correlation FITS HDU.""" + nbins = len(values) + lst = np.arange(1, nbins + 1) + + col1 = fits.Column(name="BIN1", format="K", array=np.ones(nbins)) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(nbins)) + col3 = fits.Column(name="ANGBIN", format="K", array=lst) + col4 = fits.Column(name="VALUE", format="D", array=values) + col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=theta) + + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + hdu = fits.BinTableHDU.from_columns(coldefs, name=name) + + hdu_dict = { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + } + for key, value in hdu_dict.items(): + hdu.header[key] = value + + return hdu + + +def nz_to_fits(filename): + """Convert n(z) text file to FITS format.""" + line = np.loadtxt(filename, max_rows=1) + nbins = len(line) - 1 + + z_low = np.loadtxt(filename, usecols=0) + nstep = z_low[1] - z_low[0] + + z_mid = z_low + nstep / 2 + z_high = np.append(z_low[1:], z_low[-1] + nstep) + + col1 = fits.Column(name="Z_LOW", format="D", array=z_low) + col2 = fits.Column(name="Z_MID", format="D", array=z_mid) + col3 = fits.Column(name="Z_HIGH", format="D", array=z_high) + cols = [col1, col2, col3] + + for i in range(nbins): + bin_col = np.loadtxt(filename, usecols=i + 1) + hdu_col = fits.Column(name="BIN%d" % (i + 1), format="D", array=bin_col) + cols.append(hdu_col) + + coldefs = fits.ColDefs(cols) + nz_hdu = fits.BinTableHDU.from_columns(coldefs, name="NZDATA") + + nz_lens_dict = { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": nbins, + "NZ": len(z_low), + } + + for key, value in nz_lens_dict.items(): + nz_hdu.header[key] = value + + return nz_hdu + + +def treecorr_to_fits(filename1, filename2): + """Load xi+ and xi- from separate TreeCorr FITS files.""" xiplus_hdu = fits.open(filename1) ximinus_hdu = fits.open(filename2) - return xiplus_hdu[1], ximinus_hdu[1] -def tau_to_fits(filename, theta=None): +def parse_combined_xi_fits(filepath): + """Parse combined FITS file with xip/xim columns (mock format).""" + with fits.open(filepath) as hdul: + xi_data = hdul[1].data + xip_vals = xi_data["xip"] + xim_vals = xi_data["xim"] + xi_theta = xi_data["ANG"] if "ANG" in xi_data.names else xi_data["meanr"] + + xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "P+P", "P+P") + xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "P+P", "P+P") + + return xip_hdu, xim_hdu + + +def load_glass_cl(cl_file): """ - Convert tau statistics to CosmoSIS FITS format. - - Parameters: - filename : str - Path to tau statistics FITS file - theta : array-like, optional - Angular separation values to use. If provided, overrides the theta values - from the tau statistics file. This is useful for forcing consistency with - xi correlation function angular separations. + Load GLASS mock C_ell from .npy file. + Expected shape: (5, 32) where row 0=ell, row 1=EE, row 4=BB """ - tau_stats = fits.getdata(filename) + cl_block = np.load(cl_file) + if cl_block.shape[0] < 5: + raise ValueError(f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}") - # Use provided theta if given, otherwise use tau's original theta values - if theta is not None: - ang = theta - print(f"Using provided theta values for tau statistics (forcing consistency)") - else: - ang = tau_stats["theta"] - print(f"Using original tau theta values") - - nbins = len(ang) + ell = np.asarray(cl_block[0], dtype=np.float64) + cl_ee = np.asarray(cl_block[1], dtype=np.float64) + cl_bb = np.asarray(cl_block[4], dtype=np.float64) + + return ell, cl_ee, cl_bb + + +def glass_cl_to_fits(ell, cl_ee, cl_bb): + """Convert GLASS C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" + nbins = len(ell) lst = np.arange(1, nbins + 1) - # Create fits HDU for tau_0_+ - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col1 = fits.Column(name="BIN1", format="K", array=np.ones(nbins)) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(nbins)) col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=tau_stats["tau_0_p"]) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=ang) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - tau_0_p_hdu = fits.BinTableHDU.from_columns(coldefs, name="TAU_0_PLUS") + col5 = fits.Column(name="ANG", format="D", array=ell) - # Create fits HDU for tau_2_+ - col4 = fits.Column(name="VALUE", format="D", array=tau_stats["tau_2_p"]) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - tau_2_p_hdu = fits.BinTableHDU.from_columns(coldefs, name="TAU_2_PLUS") + col4_ee = fits.Column(name="VALUE", format="D", array=cl_ee) + coldefs_ee = fits.ColDefs([col1, col2, col3, col4_ee, col5]) + cl_ee_hdu = fits.BinTableHDU.from_columns(coldefs_ee, name="CELL_EE") - # Append tau_0_p/tau_2_p header info - tau_0_p_dict = { + cl_ee_dict = { "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "P+R", + "QUANT1": "GEF", + "QUANT2": "GEF", "KERNEL_1": "NZ_SOURCE", "KERNEL_2": "NZ_SOURCE", "WINDOWS": "SAMPLE", } + for key, value in cl_ee_dict.items(): + cl_ee_hdu.header[key] = value - for key in tau_0_p_dict: - tau_0_p_hdu.header[key] = tau_0_p_dict[key] + col4_bb = fits.Column(name="VALUE", format="D", array=cl_bb) + coldefs_bb = fits.ColDefs([col1, col2, col3, col4_bb, col5]) + cl_bb_hdu = fits.BinTableHDU.from_columns(coldefs_bb, name="CELL_BB") - tau_2_p_dict = { + cl_bb_dict = { "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "SR+R", + "QUANT1": "GBF", + "QUANT2": "GBF", "KERNEL_1": "NZ_SOURCE", "KERNEL_2": "NZ_SOURCE", "WINDOWS": "SAMPLE", } + for key, value in cl_bb_dict.items(): + cl_bb_hdu.header[key] = value - for key in tau_2_p_dict: - tau_2_p_hdu.header[key] = tau_2_p_dict[key] + return cl_ee_hdu, cl_bb_hdu - return tau_0_p_hdu, tau_2_p_hdu -def pseudo_cl_to_fits(filename): - pseudo_cl = fits.getdata(filename) - cl_ee = pseudo_cl["EE"] - ell = pseudo_cl["ELL"] +def tau_to_fits(filename, theta=None): + """ + Convert tau statistics to FITS format. + If theta provided, override original values for consistency with xi. + """ + tau_stats = fits.getdata(filename) - nbins = len(ell) - lst = np.arange(1, nbins+1) - - #Create fits HDU for tau_0_+ - col1 = fits.Column(name ='BIN1', format ='K', array = np.ones(len(lst))) - col2 = fits.Column(name ='BIN2', format ='K', array = np.ones(len(lst))) - col3 = fits.Column(name ='ANGBIN', format ='K', array = lst) - col4 = fits.Column(name = 'VALUE', format = 'D', array = cl_ee) - col5 = fits.Column(name = 'ANG', format = 'D', array = ell) - coldefs = fits.ColDefs([col1,col2,col3,col4,col5]) - pseudo_cl_hdu = fits.BinTableHDU.from_columns(coldefs,name ='CELL_EE') - - pseudo_cl_dict = { - '2PTDATA': 'T', - 'QUANT1' : 'GEF', - 'QUANT2' : 'GEF', - 'KERNEL_1': 'NZ_SOURCE', - 'KERNEL_2': 'NZ_SOURCE', - 'WINDOWS': 'SAMPLE' - } + if theta is not None: + ang = theta + print("Using provided theta values for tau statistics (forcing consistency)") + else: + ang = tau_stats["theta"] + print("Using original tau theta values") - for key in pseudo_cl_dict: - pseudo_cl_hdu.header[key] = pseudo_cl_dict[key] + tau_0_p_hdu = _create_2pt_hdu(tau_stats["tau_0_p"], ang, "TAU_0_PLUS", "G+R", "P+R") + tau_2_p_hdu = _create_2pt_hdu( + tau_stats["tau_2_p"], ang, "TAU_2_PLUS", "G+R", "SR+R" + ) - return pseudo_cl_hdu + return tau_0_p_hdu, tau_2_p_hdu -def cov_pseudo_cl_to_fits(filename): - cov_pseudo_cl = fits.open(filename) - cov_ee = cov_pseudo_cl["COVAR_FULL"].data - cov_hdu = fits.ImageHDU(cov_ee) - cov_dict = { - 'COVDATA': 'True', - 'EXTNAME': 'COVMAT', - 'NAME_0': 'CELL_EE', - 'STRT_0': 0, - } +def rho_to_fits(filename, theta=None): + """ + Convert rho statistics to FITS format. + If theta provided, override original values for consistency with xi. + """ + rho_stat_hdul = fits.open(filename) + rho_stat_hdu = rho_stat_hdul[1].copy() + rho_stat_hdu.name = "RHO_STATS" - for key in cov_dict: - cov_hdu.header[key] = cov_dict[key] + if theta is not None: + print( + "Forcing rho statistics to use provided theta values (forcing consistency)" + ) + rho_stat_hdu.data = rho_stat_hdu.data.copy() + rho_stat_hdu.data["theta"] = theta + else: + print("Using original rho theta values") + + rho_stat_hdul.close() + return rho_stat_hdu - return cov_hdu -#transforms text file of CosmoCov data into covmat HDU extension def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): + """ + Convert CosmoCov covariance matrix to FITS format. - # read in cov txt data from CosmoCov + If tau covariance provided, block with xi covariance. + """ covmat_xi = np.loadtxt(filename_cov_xi) + if filename_cov_tau is not None: covmat_tau = np.load(filename_cov_tau) nbins = int(len(covmat_tau) / 3) @@ -147,282 +220,352 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): [np.zeros((len(covmat_tau), len(covmat_xi))), covmat_tau], ] ) + else: + covmat = covmat_xi if len(covmat) != len(covmat[0]): - print("Error: covmat not square!") - exit() + raise RuntimeError("Covariance matrix is not square") - else: - # create covmat ImageHDU - cov_hdu = fits.ImageHDU(covmat) - - # create header - if filename_cov_tau is None: - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT", - "NAME_0": "XI_PLUS", - "STRT_0": 0, - "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat) / 2), - } - else: - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT", - "NAME_0": "XI_PLUS", - "STRT_0": 0, - "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat_xi) / 2), - "NAME_2": "TAU_0_PLUS", - "STRT_2": len(covmat_xi), - "NAME_3": "TAU_2_PLUS", - "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), - } - for key in cov_dict: - cov_hdu.header[key] = cov_dict[key] + cov_hdu = fits.ImageHDU(covmat) + + cov_dict = { + "COVDATA": "True", + "EXTNAME": "COVMAT", + "NAME_0": "XI_PLUS", + "STRT_0": 0, + "NAME_1": "XI_MINUS", + "STRT_1": int(len(covmat) / 2), + } + + filename_cov_tau and cov_dict.update({ + "NAME_2": "TAU_0_PLUS", + "STRT_2": len(covmat_xi), + "NAME_3": "TAU_2_PLUS", + "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), + }) + + for key, value in cov_dict.items(): + cov_hdu.header[key] = value return cov_hdu -# transforms nz data (that was used in CosmoCov format) into nzdat HDU extension -def nz_to_fits(filename): +# ============================================================================ +# SECTION 2: VALIDATION & CONSISTENCY CHECKS +# ============================================================================ - line = np.loadtxt(filename, max_rows=1) - nbins = len(line) - 1 - z_low = np.loadtxt(filename, usecols=0) +def check_meanr_consistency(xi_theta, tau_theta, rho_theta, threshold=1.0): + """Check and report theta consistency across xi, tau, and rho statistics.""" + print("=" * 60) + print("MEANR CONSISTENCY CHECK") + print("=" * 60) - nstep = z_low[1] - z_low[0] + tau_diff = np.abs((tau_theta - xi_theta) / xi_theta) * 100 + rho_diff = np.abs((rho_theta - xi_theta) / xi_theta) * 100 - z_mid = z_low + nstep / 2 - z_high = np.append(z_low[1:], z_low[-1] + nstep) + print(f"Xi theta range: {xi_theta.min():.6f} - {xi_theta.max():.6f} arcmin") + print(f"Tau theta range: {tau_theta.min():.6f} - {tau_theta.max():.6f} arcmin") + print(f"Rho theta range: {rho_theta.min():.6f} - {rho_theta.max():.6f} arcmin") + print() + print(f"Max tau-xi relative difference: {tau_diff.max():.3f}%") + print(f"Mean tau-xi relative difference: {tau_diff.mean():.3f}%") + print(f"Max rho-xi relative difference: {rho_diff.max():.3f}%") + print(f"Mean rho-xi relative difference: {rho_diff.mean():.3f}%") + + if tau_diff.max() > threshold or rho_diff.max() > threshold: + print(f"\nWARNING: Meanr differences exceed {threshold}% threshold") + if tau_diff.max() > threshold: + print(f" Tau-xi: max {tau_diff.max():.3f}%") + if rho_diff.max() > threshold: + print(f" Rho-xi: max {rho_diff.max():.3f}%") + else: + print(f"✓ All differences below {threshold}% threshold") - # create hdu for histogram bin - col1 = fits.Column(name="Z_LOW", format="D", array=z_low) - col2 = fits.Column(name="Z_MID", format="D", array=z_mid) - col3 = fits.Column(name="Z_HIGH", format="D", array=z_high) - cols = [col1, col2, col3] + print("=" * 60) + print() - for i in range(nbins): - bin_col = np.loadtxt(filename, usecols=i + 1) - hdu_col = fits.Column(name="BIN%d" % (i + 1), format="D", array=bin_col) - cols.append(hdu_col) - coldefs = fits.ColDefs(cols) - nz_hdu = fits.BinTableHDU.from_columns(coldefs, name="NZDATA") +# ============================================================================ +# SECTION 3: COSMOSIS CONFIGURATION GENERATION +# ============================================================================ + + +def _generate_ini_file( + args, + template_base, + priors_file, + values_file, + suffix="", + is_harmonic=False, +): + """Generate a CosmoSIS INI configuration file from template with modifications.""" + template_path = f"cosmosis_config/{template_base}" + output_path = f"cosmosis_config/cosmosis_pipeline_{args.cosmosis_root}{suffix}.ini" + + with open(template_path, "r") as f: + config_content = f.read() + + modifications = [] + + relative_fits_file = f"data/{args.cosmosis_root}/cosmosis_{args.cosmosis_root}.fits" + default_section = ( + f"[DEFAULT]\nSCRATCH = {args.data_dir}\nFITS_FILE = {relative_fits_file}" + ) + modifications.append((r"^\[DEFAULT\]", default_section)) + + output_section = ( + f"[output]\nfilename = %(SCRATCH)s/{args.cosmosis_root}/samples_" + f"{args.cosmosis_root}{suffix}.txt" + ) + modifications.append((r"^\[output\]", output_section)) + + pipeline_section = ( + f"[pipeline]\nvalues = cosmosis_config/{values_file}\npriors = " + f"cosmosis_config/{priors_file}" + ) + modifications.append((r"^\[pipeline\]", pipeline_section)) + + if not is_harmonic: + if args.use_rho_tau: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" + "\ndata_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS\nadd_xi_sys=T" + ) + else: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" + "\ndata_sets=XI_PLUS XI_MINUS" + ) + else: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" + "\ndata_sets=CELL_EE CELL_BB" + ) - # create n(z) header - nz_lens_dict = { - "NZDATA": "T ", - "EXTNAME": "NZ_SOURCE", - "NBIN": nbins, - "NZ": len(z_low), - } + modifications.append((r"^\[2pt_like\]", like_section)) - for key in nz_lens_dict: - nz_hdu.header[key] = nz_lens_dict[key] + poly_section = f"[polychord]\npolychord_outfile_root = {args.cosmosis_root}{suffix}" + modifications.append((r"^\[polychord\]", poly_section)) - return nz_hdu + test_section = ( + f"[test]\nsave_dir = %(SCRATCH)s/best_fit/{args.cosmosis_root}{suffix}" + ) + modifications.append((r"^\[test\]", test_section)) + for pattern, replacement in modifications: + config_content = re.sub( + pattern, replacement, config_content, flags=re.MULTILINE + ) -def rho_to_fits(filename, theta=None): - """ - Convert rho statistics to CosmoSIS FITS format. - - Parameters: - filename : str - Path to rho statistics FITS file - theta : array-like, optional - Angular separation values to use. If provided, replaces the theta values - in the rho statistics file. This is useful for forcing consistency with - xi correlation function angular separations. - """ - rho_stat_hdul = fits.open(filename) - rho_stat_hdu = rho_stat_hdul[1].copy() # Create a copy to avoid modifying the original - rho_stat_hdu.name = "RHO_STATS" - - # Force rho to use provided theta if given - if theta is not None: - print(f"Forcing rho statistics to use provided theta values (forcing consistency)") - # Update the theta column in the data - rho_stat_hdu.data = rho_stat_hdu.data.copy() # Make data writable - rho_stat_hdu.data['theta'] = theta - else: - print(f"Using original rho theta values") - - rho_stat_hdul.close() # Close the original file - return rho_stat_hdu - -if __name__ == "__main__": - -#combines all the data: 2pt correlation functions from treecorr, covmat from CosmoCov (must already be combined into 1 txt file), nz txt data -#into 1 fits file to be read by CosmoSIS 2pt-likelihood function -#give file path of each of the 3 components as input, also file path of desired output FITS file -#outputs nothing, but writes a new FITS file with appropriate extensions + with open(output_path, "w") as f: + f.write(config_content) - # Print all input arguments - print("=" * 60) - print("COSMOSIS_FITTING.PY - INPUT ARGUMENTS") - print("=" * 60) - print(f"Script name: {sys.argv[0]}") - print(f"Total arguments: {len(sys.argv)}") - - arg_names = [ - "script_name", - "cosmosis_root", - "output_folder", - "nz_file", - "use_pseudo_cell", - "out_file", - ] - - for i, arg in enumerate(sys.argv): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {arg}") - print("=" * 60) - print() + config_type = "harmonic-space" if is_harmonic else "real-space" + print(f"Prepared CosmoSIS {config_type} configuration file in {output_path}") + print(f"You can now run inference with the command: cosmosis {output_path}") - cosmosis_root = sys.argv[1] - output_folder = sys.argv[2] - nz_file = sys.argv[3] - out_file = sys.argv[5] +def generate_cosmosis_config(args): + """Generate CosmoSIS INI files (real-space and optional harmonic-space).""" + if args.use_rho_tau: + template_base_realspace = "cosmosis_pipeline_A_psf.ini" + priors_file = "priors_psf.ini" + values_file = "values_psf.ini" + else: + template_base_realspace = "cosmosis_pipeline_A_ia.ini" + priors_file = "priors.ini" + values_file = "values_ia.ini" + + os.makedirs("cosmosis_config", exist_ok=True) + + _generate_ini_file( + args, + template_base_realspace, + priors_file, + values_file, + suffix="", + is_harmonic=False, + ) + + if args.cl_file: + template_base_harmonic = "cosmosis_pipeline_A_ia_cell.ini" + _generate_ini_file( + args, + template_base_harmonic, + priors_file, + values_file, + suffix="_cell", + is_harmonic=True, + ) - use_pseudo_cell = sys.argv[4] - use_pseudo_cell = True if use_pseudo_cell == 'y' else False - if use_pseudo_cell: - arg_names += ["cl_root", "gaussian_part"] +# ============================================================================ +# SECTION 4: MAIN WORKFLOW +# ============================================================================ + + +def parse_args(): + """Parse command-line arguments for unified data/mock interface.""" + parser = argparse.ArgumentParser( + description="Prepare CosmoSIS inference FITS files from real or mock data. " + "Supports multiple xi input formats (separate files, combined FITS)." + ) + + parser.add_argument( + "--cosmosis-root", type=str, required=True, help="CosmoSIS root identifier" + ) + parser.add_argument( + "--data-dir", type=str, required=True, help="Output MCMC chain directory" + ) + parser.add_argument("--nz-file", type=str, required=True, help="Path to n(z) file") + parser.add_argument( + "--out-file", type=str, required=True, help="Path to output FITS file" + ) + parser.add_argument( + "--xi", + nargs="+", + required=True, + help="Xi files: 1 (mock FITS) or 2 (data: xi_plus.fits, xi_minus.fits)", + ) + parser.add_argument( + "--cov-xi", type=str, required=True, help="Xi covariance matrix file" + ) + + parser.add_argument( + "--use-rho-tau", + action="store_true", + help="Include PSF systematics (requires --rho-stats, --tau-stats, --cov-tau)", + ) + parser.add_argument( + "--rho-stats", + type=str, + required=False, + help="Path to rho statistics FITS file (required if --use-rho-tau)", + ) + parser.add_argument( + "--tau-stats", + type=str, + required=False, + help="Path to tau statistics FITS file (required if --use-rho-tau)", + ) + parser.add_argument( + "--cov-tau", + type=str, + required=False, + help="Path to tau covariance matrix (required if --use-rho-tau)", + ) + parser.add_argument( + "--cl-file", + type=str, + required=False, + help="Path to C_ell data file (.npy, optional for data and mock)", + ) + parser.add_argument( + "--mock", action="store_true", help="Mock data mode" + ) + + return parser.parse_args() - for i, argv in enumerate(sys.argv[6:], start=6): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {argv}") - else: - print(f" argv[{i}] (unexpected): {argv}") + +if __name__ == "__main__": + args = parse_args() + + try: + print("=" * 60) + print("COSMOSIS_FITTING.PY") + print("=" * 60) + print(f"cosmosis_root: {args.cosmosis_root}") + print(f"data_dir: {args.data_dir}") + print(f"nz_file: {args.nz_file}") + print(f"out_file: {args.out_file}") + print(f"xi files: {args.xi}") + print(f"cov_xi: {args.cov_xi}") + print(f"use_rho_tau: {args.use_rho_tau}") + if args.use_rho_tau: + print(f"rho_stats: {args.rho_stats}") + print(f"tau_stats: {args.tau_stats}") + print(f"cov_tau: {args.cov_tau}") + if args.cl_file: + print(f"cl_file: {args.cl_file}") print("=" * 60) print() - cl_root = sys.argv[6] - gaussian_part = sys.argv[7] # "iNKA" or "OneCovariance" - assert gaussian_part in ["iNKA", "OneCovariance"], "gaussian_part must be either 'iNKA' or 'OneCovariance'" + if args.use_rho_tau and not all([args.rho_stats, args.tau_stats, args.cov_tau]): + raise ValueError( + "--use-rho-tau requires: --rho-stats, --tau-stats, --cov-tau" + ) - pseudo_cl_file = output_folder+'/pseudo_cl_'+cl_root+'.fits' - pseudo_cl_cov_file = output_folder+f'/pseudo_cl_cov_g_ng_{gaussian_part}_{cl_root}.fits' + os.makedirs(args.data_dir, exist_ok=True) + output_dir = os.path.dirname(args.out_file) + os.makedirs(output_dir, exist_ok=True) - print("Creating 2PT fits extension...\n") - if not os.path.exists(pseudo_cl_file): - raise FileNotFoundError("Pseudo-Cl file not found. Please run cosmo_val.py first.") - pseudo_cl_hdu = pseudo_cl_to_fits(pseudo_cl_file) - print("Creating CovMat fits extension...\n") - cov_hdu = cov_pseudo_cl_to_fits(pseudo_cl_cov_file) + print("Loading xi correlation functions...") + if args.mock: + xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) + else: + xip_hdu, xim_hdu = parse_combined_xi_fits(args.xi[0]) + + xi_theta = xip_hdu.data["ANG"] + print(f"Loaded xi: {len(xip_hdu.data)} bins") + + print("Loading covariance matrix...") + cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=None) + print(f"Loaded covariance: shape {cov_hdu.data.shape}") + + print("Loading n(z)...") + nz_hdu = nz_to_fits(args.nz_file) + print("Loaded n(z)") + + cl_ee_hdu = None + cl_bb_hdu = None + if args.cl_file: + print("Loading Cl data...") + if args.cl_file.endswith(".npy"): + ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) + cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + print(f"Loaded Cl: {len(ell)} multipoles") + else: + raise NotImplementedError(f"Cl format not supported: {args.cl_file}") + + rho_hdu = None + tau_0_p_hdu = None + tau_2_p_hdu = None + if args.use_rho_tau: + print("Loading rho/tau statistics...") + tau_stats = fits.getdata(args.tau_stats) + rho_stats = fits.getdata(args.rho_stats) + tau_theta = tau_stats["theta"] + rho_theta = rho_stats["theta"] + + check_meanr_consistency(xi_theta, tau_theta, rho_theta, threshold=5.0) + print("✓ Forcing rho and tau to use xi meanr values for consistency") - else: - arg_names += ["xi_root", "rhotau_stats", "tau_root", "covmat"] + rho_hdu = rho_to_fits(args.rho_stats, theta=xi_theta) + tau_0_p_hdu, tau_2_p_hdu = tau_to_fits(args.tau_stats, theta=xi_theta) + print("Loaded rho/tau statistics") - for i, argv in enumerate(sys.argv[6:], start=6): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {argv}") - else: - print(f" argv[{i}] (unexpected): {argv}") - print("=" * 60) + cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=args.cov_tau) + + pri_hdr = fits.Header() + pri_hdu = fits.PrimaryHDU(header=pri_hdr) + + print("Assembling FITS file...") + hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] + + if args.cl_file: + hdu_list.extend([cl_ee_hdu, cl_bb_hdu]) + + if args.use_rho_tau: + hdu_list.extend([tau_0_p_hdu, tau_2_p_hdu, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(args.out_file, overwrite=True) + print(f"✓ FITS file written to {args.out_file}") print() - xi_root = sys.argv[6] - rhotau_stats = sys.argv[7] - tau_root = sys.argv[8] - cov_xi_file = sys.argv[9] #in cosmocov combined txt format - - two_pt_file_xip = output_folder+'/xi_plus_'+xi_root+'.fits' - two_pt_file_xim = output_folder+'/xi_minus_'+xi_root+'.fits' - #in cosmocov format - - use_tau_stats = sys.argv[7] - use_tau_stats = True if use_tau_stats == 'y' else False - rho_stats_file = output_folder+'/rho_tau_stats/rho_stats_'+tau_root+'.fits' if use_tau_stats else None - tau_stats_file = output_folder +'/rho_tau_stats/tau_stats_'+tau_root+'.fits' if use_tau_stats else None - cov_tau_file = output_folder + '/rho_tau_stats/cov_tau_'+tau_root+'_th.npy' if use_tau_stats else None - - - #create the required FITS extensions - print("Creating 2PT fits extension...\n") - if not (os.path.exists(two_pt_file_xip) and os.path.exists(two_pt_file_xim)): - raise FileNotFoundError("2pt files not found. Please run cosmo_val.py first.") - xip_hdu, xim_hdu = treecorr_to_fits(two_pt_file_xip, two_pt_file_xim) - - # Extract xi meanr for consistency enforcement - xi_theta = xip_hdu.data['ANG'] # xi uses 'ANG' column for meanr - - if use_tau_stats and not os.path.exists(tau_stats_file): - raise Warning("Tau stats file not found. Please run cosmo_val.py first. Creating the FITS file without rho and tau statistics.") - use_tau_stats= False - cov_tau_file = None - if use_tau_stats: - - # Load original theta values for validation - tau_stats = fits.getdata(str(tau_stats_file)) - rho_stats = fits.getdata(str(rho_stats_file)) - tau_theta = tau_stats['theta'] - rho_theta = rho_stats['theta'] - - # Validate theta consistency and report differences - print("=" * 60) - print("MEANR CONSISTENCY CHECK") - print("=" * 60) - - # Calculate relative differences - tau_diff = np.abs((tau_theta - xi_theta) / xi_theta) * 100 - rho_diff = np.abs((rho_theta - xi_theta) / xi_theta) * 100 - - print(f"Xi theta range: {xi_theta.min():.6f} - {xi_theta.max():.6f} arcmin") - print(f"Tau theta range: {tau_theta.min():.6f} - {tau_theta.max():.6f} arcmin") - print(f"Rho theta range: {rho_theta.min():.6f} - {rho_theta.max():.6f} arcmin") - print() - print(f"Max tau-xi relative difference: {tau_diff.max():.3f}%") - print(f"Mean tau-xi relative difference: {tau_diff.mean():.3f}%") - print(f"Max rho-xi relative difference: {rho_diff.max():.3f}%") - print(f"Mean rho-xi relative difference: {rho_diff.mean():.3f}%") - - # Check for excessive differences - max_allowed_diff = 5.0 # 5% threshold - if tau_diff.max() > max_allowed_diff: - raise ValueError(f"Tau-xi meanr difference exceeds {max_allowed_diff}%: {tau_diff.max():.3f}%") - if rho_diff.max() > max_allowed_diff: - raise ValueError(f"Rho-xi meanr difference exceeds {max_allowed_diff}%: {rho_diff.max():.3f}%") - - print(f"✓ All differences below {max_allowed_diff}% threshold") - print("✓ Forcing rho and tau to use xi meanr values for consistency") - print("=" * 60) - print() - - print('Creating rho stats fits extension...\n') - rho_hdu = rho_to_fits(rho_stats_file) - print("Creating tau fits extensions...\n") - tau_0_p_hdu, tau_2_p_hdu = tau_to_fits(tau_stats_file) - print("Creating CovMat fits extension...\n") - cov_hdu = covdat_to_fits(cov_xi_file, cov_tau_file) - print("Creating n(z) fits extension...\n") - nz_hdu = nz_to_fits(nz_file) - - # create header for primary HDU - pri_hdr_dict = {} - - pri_hdr = fits.Header() - for key in pri_hdr_dict: - pri_hdr[key] = pri_hdr_dict[key] - - # create primary HDU - pri_hdu = fits.PrimaryHDU(header=pri_hdr) - - # create final FITS HDU - print("Writing out combined FITS file...\n") - if use_pseudo_cell: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, pseudo_cl_hdu] - else: - if use_tau_stats: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu, tau_0_p_hdu, tau_2_p_hdu, rho_hdu] - else: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] - hdul = fits.HDUList(hdu_list) - hdul.writeto(out_file, overwrite=True) - print("FITS file written out to %s" % out_file) + generate_cosmosis_config(args) + + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) From ecf3769f20d6b1de2e72ced847b6cbf99c1d14e2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 27 Oct 2025 23:39:51 +0100 Subject: [PATCH 19/29] remove defunct pipeline.sh --- cosmo_inference/pipeline.sh | 127 ------------------------------------ 1 file changed, 127 deletions(-) delete mode 100755 cosmo_inference/pipeline.sh diff --git a/cosmo_inference/pipeline.sh b/cosmo_inference/pipeline.sh deleted file mode 100755 index 8d20cb9..0000000 --- a/cosmo_inference/pipeline.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# Transform long options to short ones -for arg in "$@"; do - shift - case "$arg" in - '--help') set -- "$@" '-h' ;; - '--pcf') set -- "$@" '-p' ;; - '--covmat') set -- "$@" '-c' ;; - '--inference') set -- "$@" '-i' ;; - '--mcmc_process') set -- "$@" '-m' ;; - *) set -- "$@" "$arg" ;; - esac -done - -# Parse short options -OPTIND=1 -while getopts "hpcim" opt -do - case "$opt" in - 'h') - echo "Please input a flag: --help, --pcf, --covmat, --inference or --mcmc_process "; - exit 0 - ;; - 'p') - echo "Running cosmo_val.py to calculate 2 point correlation functions"; - python notebooks/cosmo_val/cosmo_val.py - ;; - 'c') - read -p 'COVARIANCE FILE: ' covmat_file; - read -p 'OUTPUT STUB (without extension): ' output_stub; - echo "Processing covariance matrix"; - python scripts/cosmocov_process.py $covmat_file $output_stub - ;; - 'i') - read -p 'COSMOSIS ROOT: ' cosmosis_root; - - # Create the folder if it does not exist - if [ ! -d "data/$cosmosis_root" ]; then - mkdir -p "data/$cosmosis_root" - echo "Directory 'data/$cosmosis_root' created." - else - echo "Directory 'data/$cosmosis_root' already exists." - fi - - read -p 'COSMO_VAL OUTPUT FOLDER: ' output_folder; - read -p 'NZ FILE:' nz_file; - read -p 'OUTPUT MCMC CHAIN FOLDER: ' data; - read -p 'USE PSEUDO_CELL? (y/n): ' pseudo_cell; - - if [ "${pseudo_cell}" == "y" ]; then - - read -p 'CL ROOT: ' cl_root; - read -p 'GAUSSIAN COVARIANCE? (iNKA/OneCovariance): ' gaussian_part; - - - echo "Using pseudo cell" - out_file="data/${cosmosis_root}/cosmosis_${cosmosis_root}_cell.fits" - - python scripts/cosmosis_fitting.py $cosmosis_root $output_folder $nz_file $pseudo_cell $out_file $cl_root $gaussian_part; - else - read -p 'XI ROOT: ' xi_root; - read -p 'USE RHO/TAU_STATS? (y/n): ' rhotau_stats; - echo $rhotau_stats - read -p 'TAU ROOT: ' tau_root; - read -p 'COV_XI MAT TXT FILE:' covmat; - - out_file="data/${cosmosis_root}/cosmosis_${cosmosis_root}.fits"; - - #LG: add check if xi_plus/xi_minus fits file exists - python scripts/cosmosis_fitting.py $cosmosis_root $output_folder $nz_file $pseudo_cell $out_file $xi_root $rhotau_stats $tau_root $covmat; - - fi - - if [ "${pseudo_cell}" == "y" ]; then - output_ini_file="cosmosis_config/cosmosis_pipeline_${cosmosis_root}_cell.ini" - cp cosmosis_config/cosmosis_pipeline_A_ia_cell.ini $output_ini_file - else - output_ini_file="cosmosis_config/cosmosis_pipeline_${cosmosis_root}.ini" - if [ "${rhotau_stats}" == "y" ]; then - cp cosmosis_config/cosmosis_pipeline_A_psf.ini $output_ini_file; - else - cp cosmosis_config/cosmosis_pipeline_A_ia.ini $output_ini_file; - fi - fi - - sed -i "/^\[DEFAULT\]/a\SCRATCH = ${data}" $output_ini_file; - sed -i "/^\[DEFAULT\]/a\FITS_FILE = ${out_file}" $output_ini_file; - if [ "${pseudo_cell}" == "y" ]; then - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${cosmosis_root}_cell/samples_${cosmosis_root}_cell.txt" $output_ini_file; - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=CELL_EE" $output_ini_file; - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${cosmosis_root}_cell" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${cosmosis_root}_cell" $output_ini_file; - else - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${cosmosis_root}/samples_${cosmosis_root}.txt" $output_ini_file; - if [ "${rhotau_stats}" == "y" ]; then - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_psf.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors_psf.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS" $output_ini_file; - sed -i "/^\[2pt_like]/a\add_xi_sys=T" $output_ini_file; - else - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS" $output_ini_file; - fi - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}" $output_ini_file; - fi - echo "Prepared CosmoSIS configuration file in $output_ini_file"; - echo "You can now run the inference with the command: cosmosis $output_ini_file" - ;; - 'm') - # LG: also convert this into a script to directly output contour plots - echo "Run the cosmo_inference/notebooks/MCMC.ipynb notebook to analyse your chains" - ;; - '?') - print_usage >&2; - exit 1 - ;; - esac -done -shift $(expr $OPTIND - 1) # remove options from positional parameters \ No newline at end of file From e76ba2c16a74d2329b0e5aeea827b27932c18bc2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 00:21:33 +0100 Subject: [PATCH 20/29] add example call --- cosmo_inference/scripts/cosmosis_fitting.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 6c2fc8f..f0328d3 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -409,7 +409,23 @@ def parse_args(): """Parse command-line arguments for unified data/mock interface.""" parser = argparse.ArgumentParser( description="Prepare CosmoSIS inference FITS files from real or mock data. " - "Supports multiple xi input formats (separate files, combined FITS)." + "Supports multiple xi input formats (separate files, combined FITS).", + epilog=""" +Example for SP v1.4.6_A: + python cosmo_inference/scripts/cosmosis_fitting.py \\ + --cosmosis-root "SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --data-dir "/n09data/guerrini/output_chains" \\ + --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ + --out-file "cosmo_inference/data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --xi "notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + "notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-xi "cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_processed.txt" \\ + --use-rho-tau \\ + --rho-stats "notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-tau "notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" + """, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument( From ddfb617dfa75cc9d5ce8946da74536e317632b35 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:35:59 +0100 Subject: [PATCH 21/29] bugfix and readme --- cosmo_inference/README.md | 38 +++++++++++++++------ cosmo_inference/scripts/cosmosis_fitting.py | 10 ++++-- src/sp_validation/rho_tau.py | 18 +++++----- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 4da2aaa..eec8043 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -7,20 +7,38 @@ This folder contains the files neccessary to run the cosmological inference pipe To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) and [CosmoCov](https://github.com/CosmoLike/CosmoCov). To PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run -Run the bash script within this folder +The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: +```bash +snakemake --profile slurm -j inference_fiducial ``` -$ ./pipeline.sh -``` -with one of the following flags: - -`--pcf`: This step runs the `cosmo_val.py` script to calculate the various 2 point correlation functions. It will also write the $\xi_{pm}$ correlation functions in a fits file. -`--covmat`: The covariance matrix is calculated here using CosmoCov, by reading in the `./cosmocov_config/cosmocov_{output_root}.ini` file. **Hence make sure the `output_root` here corresponds to the one entered in the prompt**. - -`--inference`: This step writes out the relevant `./cosmosis_config/cosmosis_{output_root}.ini` file, in order to run CosmoSIS to conduct the cosmological inference. It also combines the data needed by CosmoSIS: the $\xi_{pm}$ fits files calculated in `cosmo_val.py`, the covariance matrix, and the nz catalogue, into a single `.fits` file. It also fetches the rho-statistics computed in `cosmo_val.py` to marginalize on PSF leakage parameters. To do so, specify the path to the `cosmo_val` outputs. Note that the Rho- and Tau-statistics have to be stored in a folder `rho_tau_stats` in this folder. If the 2PCF $\xi_{pm}$ are not found it will raise an `Error` to run `cosmo_val`. If the Rho- and Tau-statistics do not exist, the script will raise a `Warning` to run `cosmo_val` but will create a data vector without the Rho- and Tau-statistics. +This will automatically execute all steps: +1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` +2. Compute covariance matrices using CosmoCov +3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` +4. Run CosmoSIS inference + +For standalone FITS data preparation, you can also use the Python script directly: + +```bash +python scripts/cosmosis_fitting.py \ + --cosmosis-root "catalog_version_config" \ + --data-dir "/path/to/output/chains" \ + --nz-file "/path/to/nz_file.txt" \ + --out-file "/path/to/output.fits" \ + --xi "/path/to/xi_plus.fits" "/path/to/xi_minus.fits" \ + --cov-xi "/path/to/covariance.txt" \ + --use-rho-tau \ + --rho-stats "/path/to/rho_stats.fits" \ + --tau-stats "/path/to/tau_stats.fits" \ + --cov-tau "/path/to/cov_tau.npy" +``` -`--mcmc_process`: You can finally analyse the chains with the `MCMC.ipynb` notebook. +You can view all available options with: +```bash +python scripts/cosmosis_fitting.py --help +``` This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. \ No newline at end of file diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index f0328d3..97659dd 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -234,7 +234,7 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): "NAME_0": "XI_PLUS", "STRT_0": 0, "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat) / 2), + "STRT_1": int(len(covmat_xi) / 2), } filename_cov_tau and cov_dict.update({ @@ -518,9 +518,13 @@ def parse_args(): print("Loading xi correlation functions...") if args.mock: - xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) - else: + # Mock mode expects a single combined FITS file xip_hdu, xim_hdu = parse_combined_xi_fits(args.xi[0]) + else: + # Data mode expects two TreeCorr files (xi_plus, xi_minus) + if len(args.xi) != 2: + raise ValueError(f"Data mode requires exactly 2 xi files, got {len(args.xi)}") + xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) xi_theta = xip_hdu.data["ANG"] print(f"Loaded xi: {len(xip_hdu.data)} bins") diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 2647b37..2ba9ee4 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -597,12 +597,12 @@ def get_samples_emcee( psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") - # Check if the path exists - sample_file_path = psf_fitter.get_sample_file_path(version) + # Check if the path exists (use base for cache key to account for different TreeCorr configs) + sample_file_path = psf_fitter.get_sample_file_path(base) if os.path.exists(sample_file_path): print(f"Skipping sampling; {sample_file_path} exists.") - flat_samples = psf_fitter.load_samples(version) - mcmc_result, q = psf_fitter.get_mcmc_from_samples(version) + flat_samples = psf_fitter.load_samples(base) + mcmc_result, q = psf_fitter.get_mcmc_from_samples(base) print(mcmc_result) # Or run MCMC else: @@ -618,7 +618,7 @@ def get_samples_emcee( apply_debias=debias_npatch is not None, savefig="mcmc_samples_" + version + ".png", ) - psf_fitter.save_samples(flat_samples, version) + psf_fitter.save_samples(flat_samples, base) return flat_samples, mcmc_result, q @@ -648,11 +648,11 @@ def get_samples_lsq( psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") - # Check if the path exists - sample_file_path = psf_fitter.get_sample_path(version) + # Check if the path exists (use base for cache key to account for different TreeCorr configs) + sample_file_path = psf_fitter.get_sample_path(base) if os.path.exists(sample_file_path): print(f"Skipping sampling; {sample_file_path} exists.") - flat_samples = psf_fitter.load_samples(version) + flat_samples = psf_fitter.load_samples(base) mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC @@ -666,5 +666,5 @@ def get_samples_lsq( flat_samples, mcmc_result, q = psf_fitter.get_least_squares_params_samples( npatch=debias_npatch, apply_debias=(debias_npatch is not None) ) - psf_fitter.save_samples(flat_samples, version) + psf_fitter.save_samples(flat_samples, base) return flat_samples, mcmc_result, q From a24828259b5a5cbdb2213feb8efb4e68da475720 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:36:20 +0100 Subject: [PATCH 22/29] example ini files for glass mock; not using new priors etc. yet --- .../cosmosis_pipeline_glass_mock_00001.ini | 124 ++++++++++++++++++ ...osmosis_pipeline_glass_mock_00001_cell.ini | 113 ++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini new file mode 100644 index 0000000..ffc0a37 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/glass_mock_chains/glass_mock_00001 +FITS_FILE = data/glass_mock_00001/cosmosis_glass_mock_00001.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/glass_mock_00001 + +[polychord] +polychord_outfile_root = glass_mock_00001 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/glass_mock_00001/samples_glass_mock_00001.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini new file mode 100644 index 0000000..ccb6ecc --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/glass_mock_chains/glass_mock_00001 +FITS_FILE = data/glass_mock_00001/cosmosis_glass_mock_00001.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/glass_mock_00001_cell + + +[polychord] +polychord_outfile_root = glass_mock_00001_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/glass_mock_00001/samples_glass_mock_00001_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE CELL_BB +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like \ No newline at end of file From 55b071968842ed7923cc533a2a01380e550848e3 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:43:06 +0100 Subject: [PATCH 23/29] smarter gitignore --- .gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2709b40..9b266a6 100644 --- a/.gitignore +++ b/.gitignore @@ -173,5 +173,12 @@ notebooks/demo_add_bands.ipynb CLAUDE.md AGENTS.md -cosmo_inference/cosmosis_config/cosmosis_pipeline_S*.ini +cosmo_inference/cosmosis_config/*.ini cosmo_inference/cosmosis_config/priors_S*.ini +!cosmosis_config/cosmosis_pipeline_A_ia_cell.ini +!cosmosis_config/cosmosis_pipeline_A_ia.ini +!cosmosis_config/cosmosis_pipeline_A_psf.ini +!cosmosis_config/cosmosis_pipeline_A.ini +!cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +!cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +!cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file From 0a937c7403773518fd46ea7308465c8b98aec171 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:45:23 +0100 Subject: [PATCH 24/29] fix gitignore, add data ini --- .gitignore | 15 +-- ...sep=1.0_maxsep=250.0_nbins=20_npatch=1.ini | 124 ++++++++++++++++++ 2 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini diff --git a/.gitignore b/.gitignore index 9b266a6..fcd96fc 100644 --- a/.gitignore +++ b/.gitignore @@ -174,11 +174,10 @@ notebooks/demo_add_bands.ipynb CLAUDE.md AGENTS.md cosmo_inference/cosmosis_config/*.ini -cosmo_inference/cosmosis_config/priors_S*.ini -!cosmosis_config/cosmosis_pipeline_A_ia_cell.ini -!cosmosis_config/cosmosis_pipeline_A_ia.ini -!cosmosis_config/cosmosis_pipeline_A_psf.ini -!cosmosis_config/cosmosis_pipeline_A.ini -!cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini -!cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini -!cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini new file mode 100644 index 0000000..d64ac64 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file From 508a6520eefa3da9911dabb4a9eba766b706225d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 28 Oct 2025 01:59:08 +0100 Subject: [PATCH 25/29] make sure cls are included in data fits file --- .gitignore | 3 +- ....0_maxsep=250.0_nbins=20_npatch=1_cell.ini | 113 ++++++++++++++++++ cosmo_inference/scripts/cosmosis_fitting.py | 16 +++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini diff --git a/.gitignore b/.gitignore index fcd96fc..156b88e 100644 --- a/.gitignore +++ b/.gitignore @@ -180,4 +180,5 @@ cosmo_inference/cosmosis_config/*.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_A.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini -!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini \ No newline at end of file +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini new file mode 100644 index 0000000..3d28be3 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell + + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE CELL_BB +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 97659dd..32bd02d 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -115,6 +115,18 @@ def load_glass_cl(cl_file): return ell, cl_ee, cl_bb +def load_pseudo_cl_fits(cl_file): + """ + Load pseudo-C_ell data stored in FITS format with columns (ELL, EE, EB, BB). + """ + with fits.open(cl_file) as hdul: + data = hdul[1].data + ell = np.asarray(data["ELL"], dtype=np.float64) + cl_ee = np.asarray(data["EE"], dtype=np.float64) + cl_bb = np.asarray(data["BB"], dtype=np.float64) + return ell, cl_ee, cl_bb + + def glass_cl_to_fits(ell, cl_ee, cl_bb): """Convert GLASS C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" nbins = len(ell) @@ -545,6 +557,10 @@ def parse_args(): ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) print(f"Loaded Cl: {len(ell)} multipoles") + elif args.cl_file.endswith(".fits"): + ell, cl_ee, cl_bb = load_pseudo_cl_fits(args.cl_file) + cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + print(f"Loaded Cl: {len(ell)} multipoles (FITS pseudo-Cl)") else: raise NotImplementedError(f"Cl format not supported: {args.cl_file}") From f9a81792af6b300bc1c4a00bad4d26ac584d4469 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 29 Oct 2025 03:50:28 +0100 Subject: [PATCH 26/29] Update inference configs and pseudo-Cl handling --- .gitignore | 3 +- cosmo_inference/README.md | 10 ++- ....0_maxsep=250.0_nbins=20_npatch=1_cell.ini | 4 +- .../cosmosis_pipeline_glass_mock_00001.ini | 2 +- ...osmosis_pipeline_glass_mock_00001_cell.ini | 6 +- .../cosmosis_config/priors_mock.ini | 11 +++ cosmo_inference/scripts/cosmosis_fitting.py | 80 ++++++++++++++++--- 7 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/priors_mock.ini diff --git a/.gitignore b/.gitignore index 156b88e..3208c72 100644 --- a/.gitignore +++ b/.gitignore @@ -181,4 +181,5 @@ cosmo_inference/cosmosis_config/*.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini -!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini \ No newline at end of file +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini +!cosmo_inference/cosmosis_config/priors_mock.ini diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index eec8043..f729285 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -19,7 +19,7 @@ This will automatically execute all steps: 3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` 4. Run CosmoSIS inference -For standalone FITS data preparation, you can also use the Python script directly: +For standalone FITS data preparation (real-space inputs plus optional pseudo-$C_\ell$ data), you can also use the Python script directly: ```bash python scripts/cosmosis_fitting.py \ @@ -32,7 +32,9 @@ python scripts/cosmosis_fitting.py \ --use-rho-tau \ --rho-stats "/path/to/rho_stats.fits" \ --tau-stats "/path/to/tau_stats.fits" \ - --cov-tau "/path/to/cov_tau.npy" + --cov-tau "/path/to/cov_tau.npy" \ + --cl-file "/path/to/pseudo_cl.fits" \ + --cov-cl "/path/to/pseudo_cl_cov.fits" ``` You can view all available options with: @@ -40,5 +42,7 @@ You can view all available options with: python scripts/cosmosis_fitting.py --help ``` +Ensure the pseudo-$C_\ell$ spectra (`pseudo_cl_*.fits`) and their covariance (`pseudo_cl_cov_*.fits`) produced by `cosmo_val.py` exist for the requested catalog version (or mock seed) before running the standalone command. -This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. \ No newline at end of file + +This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini index 3d28be3..34a7209 100644 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini @@ -105,9 +105,9 @@ corr_type = 0 ; shear_cl -> shear_xi [2pt_like] file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE CELL_BB +data_sets=CELL_EE data_file=%(FITS_FILE)s gaussian_covariance=F -covmat_name=COVMAT +covmat_name=COVMAT_CELL cut_zeros=F like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini index ffc0a37..85374a2 100644 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini @@ -7,7 +7,7 @@ COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library [pipeline] values = cosmosis_config/values_psf.ini -priors = cosmosis_config/priors_psf.ini +priors = cosmosis_config/priors_mock.ini modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like likelihoods = 2pt_like extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini index ccb6ecc..f1a03fd 100644 --- a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini @@ -7,7 +7,7 @@ COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library [pipeline] values = cosmosis_config/values_psf.ini -priors = cosmosis_config/priors_psf.ini +priors = cosmosis_config/priors_mock.ini modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like likelihoods = 2pt_like extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m @@ -105,9 +105,9 @@ corr_type = 0 ; shear_cl -> shear_xi [2pt_like] file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py -data_sets=CELL_EE CELL_BB +data_sets=CELL_EE data_file=%(FITS_FILE)s gaussian_covariance=F -covmat_name=COVMAT +covmat_name=COVMAT_CELL cut_zeros=F like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_mock.ini b/cosmo_inference/cosmosis_config/priors_mock.ini new file mode 100644 index 0000000..0fe8933 --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_mock.ini @@ -0,0 +1,11 @@ +[intrinsic_alignment_parameters] +A = gaussian 0.83 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian 0.0 0.01 + +[nofz_shifts] +bias_1 = gaussian -0.033 0.013 diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 32bd02d..38a88b0 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -93,8 +93,8 @@ def parse_combined_xi_fits(filepath): xim_vals = xi_data["xim"] xi_theta = xi_data["ANG"] if "ANG" in xi_data.names else xi_data["meanr"] - xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "P+P", "P+P") - xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "P+P", "P+P") + xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "G-R", "G-R") return xip_hdu, xim_hdu @@ -127,8 +127,8 @@ def load_pseudo_cl_fits(cl_file): return ell, cl_ee, cl_bb -def glass_cl_to_fits(ell, cl_ee, cl_bb): - """Convert GLASS C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" +def cl_to_fits(ell, cl_ee, cl_bb): + """Convert C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" nbins = len(ell) lst = np.arange(1, nbins + 1) @@ -170,6 +170,40 @@ def glass_cl_to_fits(ell, cl_ee, cl_bb): return cl_ee_hdu, cl_bb_hdu +def cov_cl_to_fits(cov_file, nbins): + """Convert pseudo-C_ell covariance to a CosmoSIS ImageHDU.""" + if cov_file.endswith(".fits"): + with fits.open(cov_file) as hdul: + if "COVAR_EE_EE" in hdul: + cov_data = np.asarray(hdul["COVAR_EE_EE"].data, dtype=np.float64) + else: + cov_data = np.asarray(hdul[0].data, dtype=np.float64) + elif cov_file.endswith(".npy"): + cov_data = np.load(cov_file) + else: + raise NotImplementedError(f"Unsupported pseudo-Cl covariance format: {cov_file}") + + if cov_data.shape[0] != cov_data.shape[1]: + raise ValueError("Pseudo-Cl covariance matrix must be square") + if cov_data.shape[0] != nbins: + raise ValueError( + "Pseudo-Cl covariance dimension does not match C_ell data length" + ) + + cov_hdu = fits.ImageHDU(cov_data, name="COVMAT_CELL") + cov_dict = { + "COVDATA": "True", + "EXTNAME": "COVMAT_CELL", + "NAME_0": "CELL_EE", + "STRT_0": 0, + } + + for key, value in cov_dict.items(): + cov_hdu.header[key] = value + + return cov_hdu + + def tau_to_fits(filename, theta=None): """ Convert tau statistics to FITS format. @@ -352,10 +386,13 @@ def _generate_ini_file( else: like_section = ( "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" - "\ndata_sets=CELL_EE CELL_BB" + "\ndata_sets=CELL_EE" ) + covmat_line = "covmat_name=COVMAT_CELL" if is_harmonic else "covmat_name=COVMAT" + modifications.append((r"^\[2pt_like\]", like_section)) + modifications.append((r"^covmat_name=.*", covmat_line)) poly_section = f"[polychord]\npolychord_outfile_root = {args.cosmosis_root}{suffix}" modifications.append((r"^\[polychord\]", poly_section)) @@ -382,13 +419,18 @@ def generate_cosmosis_config(args): """Generate CosmoSIS INI files (real-space and optional harmonic-space).""" if args.use_rho_tau: template_base_realspace = "cosmosis_pipeline_A_psf.ini" - priors_file = "priors_psf.ini" values_file = "values_psf.ini" else: template_base_realspace = "cosmosis_pipeline_A_ia.ini" - priors_file = "priors.ini" values_file = "values_ia.ini" + if args.mock: + priors_file = "priors_mock.ini" + elif args.use_rho_tau: + priors_file = "priors_psf.ini" + else: + priors_file = "priors.ini" + os.makedirs("cosmosis_config", exist_ok=True) _generate_ini_file( @@ -489,6 +531,12 @@ def parse_args(): required=False, help="Path to C_ell data file (.npy, optional for data and mock)", ) + parser.add_argument( + "--cov-cl", + type=str, + required=False, + help="Path to pseudo-C_ell covariance matrix (required if --cl-file)", + ) parser.add_argument( "--mock", action="store_true", help="Mock data mode" ) @@ -516,6 +564,8 @@ def parse_args(): print(f"cov_tau: {args.cov_tau}") if args.cl_file: print(f"cl_file: {args.cl_file}") + if args.cov_cl: + print(f"cov_cl: {args.cov_cl}") print("=" * 60) print() @@ -523,6 +573,10 @@ def parse_args(): raise ValueError( "--use-rho-tau requires: --rho-stats, --tau-stats, --cov-tau" ) + if args.cl_file and not args.cov_cl: + raise ValueError("--cov-cl is required when --cl-file is provided") + if args.cov_cl and not args.cl_file: + raise ValueError("--cl-file is required when --cov-cl is provided") os.makedirs(args.data_dir, exist_ok=True) output_dir = os.path.dirname(args.out_file) @@ -551,18 +605,21 @@ def parse_args(): cl_ee_hdu = None cl_bb_hdu = None + cov_cl_hdu = None if args.cl_file: print("Loading Cl data...") if args.cl_file.endswith(".npy"): ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) - cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) print(f"Loaded Cl: {len(ell)} multipoles") elif args.cl_file.endswith(".fits"): ell, cl_ee, cl_bb = load_pseudo_cl_fits(args.cl_file) - cl_ee_hdu, cl_bb_hdu = glass_cl_to_fits(ell, cl_ee, cl_bb) + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) print(f"Loaded Cl: {len(ell)} multipoles (FITS pseudo-Cl)") else: raise NotImplementedError(f"Cl format not supported: {args.cl_file}") + cov_cl_hdu = cov_cl_to_fits(args.cov_cl, len(ell)) + print("Loaded pseudo-Cl covariance") rho_hdu = None tau_0_p_hdu = None @@ -587,7 +644,10 @@ def parse_args(): pri_hdu = fits.PrimaryHDU(header=pri_hdr) print("Assembling FITS file...") - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] + hdu_list = [pri_hdu, cov_hdu] + if cov_cl_hdu is not None: + hdu_list.append(cov_cl_hdu) + hdu_list.extend([nz_hdu, xip_hdu, xim_hdu]) if args.cl_file: hdu_list.extend([cl_ee_hdu, cl_bb_hdu]) From e74d3cf49d20800b9bfc69985b7d7e1ab75e5166 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 3 Nov 2025 10:24:32 +0100 Subject: [PATCH 27/29] cosmosis_fitting fixes --- cosmo_inference/scripts/cosmosis_fitting.py | 70 ++++++++------------- 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 38a88b0..350684c 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -99,32 +99,28 @@ def parse_combined_xi_fits(filepath): return xip_hdu, xim_hdu -def load_glass_cl(cl_file): - """ - Load GLASS mock C_ell from .npy file. - Expected shape: (5, 32) where row 0=ell, row 1=EE, row 4=BB - """ - cl_block = np.load(cl_file) - if cl_block.shape[0] < 5: - raise ValueError(f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}") - - ell = np.asarray(cl_block[0], dtype=np.float64) - cl_ee = np.asarray(cl_block[1], dtype=np.float64) - cl_bb = np.asarray(cl_block[4], dtype=np.float64) - - return ell, cl_ee, cl_bb +def load_pseudo_cl(cl_file): + """Load pseudo-C_ell spectra from .npy or FITS formats.""" + if cl_file.endswith(".npy"): + cl_block = np.load(cl_file) + if cl_block.shape[0] < 5: + raise ValueError( + f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}" + ) + ell = np.asarray(cl_block[0], dtype=np.float64) + cl_ee = np.asarray(cl_block[1], dtype=np.float64) + cl_bb = np.asarray(cl_block[4], dtype=np.float64) + return ell, cl_ee, cl_bb + if cl_file.endswith(".fits"): + with fits.open(cl_file) as hdul: + data = hdul[1].data + ell = np.asarray(data["ELL"], dtype=np.float64) + cl_ee = np.asarray(data["EE"], dtype=np.float64) + cl_bb = np.asarray(data["BB"], dtype=np.float64) + return ell, cl_ee, cl_bb -def load_pseudo_cl_fits(cl_file): - """ - Load pseudo-C_ell data stored in FITS format with columns (ELL, EE, EB, BB). - """ - with fits.open(cl_file) as hdul: - data = hdul[1].data - ell = np.asarray(data["ELL"], dtype=np.float64) - cl_ee = np.asarray(data["EE"], dtype=np.float64) - cl_bb = np.asarray(data["BB"], dtype=np.float64) - return ell, cl_ee, cl_bb + raise NotImplementedError(f"Cl format not supported: {cl_file}") def cl_to_fits(ell, cl_ee, cl_bb): @@ -170,14 +166,11 @@ def cl_to_fits(ell, cl_ee, cl_bb): return cl_ee_hdu, cl_bb_hdu -def cov_cl_to_fits(cov_file, nbins): +def cov_cl_to_fits(cov_file, cov_hdu="COVAR_FULL"): """Convert pseudo-C_ell covariance to a CosmoSIS ImageHDU.""" if cov_file.endswith(".fits"): with fits.open(cov_file) as hdul: - if "COVAR_EE_EE" in hdul: - cov_data = np.asarray(hdul["COVAR_EE_EE"].data, dtype=np.float64) - else: - cov_data = np.asarray(hdul[0].data, dtype=np.float64) + cov_data = np.asarray(hdul[cov_hdu].data, dtype=np.float64) elif cov_file.endswith(".npy"): cov_data = np.load(cov_file) else: @@ -185,10 +178,6 @@ def cov_cl_to_fits(cov_file, nbins): if cov_data.shape[0] != cov_data.shape[1]: raise ValueError("Pseudo-Cl covariance matrix must be square") - if cov_data.shape[0] != nbins: - raise ValueError( - "Pseudo-Cl covariance dimension does not match C_ell data length" - ) cov_hdu = fits.ImageHDU(cov_data, name="COVMAT_CELL") cov_dict = { @@ -608,17 +597,10 @@ def parse_args(): cov_cl_hdu = None if args.cl_file: print("Loading Cl data...") - if args.cl_file.endswith(".npy"): - ell, cl_ee, cl_bb = load_glass_cl(args.cl_file) - cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) - print(f"Loaded Cl: {len(ell)} multipoles") - elif args.cl_file.endswith(".fits"): - ell, cl_ee, cl_bb = load_pseudo_cl_fits(args.cl_file) - cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) - print(f"Loaded Cl: {len(ell)} multipoles (FITS pseudo-Cl)") - else: - raise NotImplementedError(f"Cl format not supported: {args.cl_file}") - cov_cl_hdu = cov_cl_to_fits(args.cov_cl, len(ell)) + ell, cl_ee, cl_bb = load_pseudo_cl(args.cl_file) + print(f"Loaded Cl: {len(ell)} multipoles") + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) + cov_cl_hdu = cov_cl_to_fits(args.cov_cl, cov_hdu="COVAR_FULL") print("Loaded pseudo-Cl covariance") rho_hdu = None From f18d6dda85490de01f47207d7659d6cb60c9ab03 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 3 Nov 2025 16:09:41 +0100 Subject: [PATCH 28/29] better control of output files --- cosmo_inference/scripts/cosmosis_fitting.py | 103 +++++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 350684c..8ad2167 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -4,6 +4,7 @@ import os import re import sys +from pathlib import Path import numpy as np from astropy.io import fits @@ -335,15 +336,15 @@ def _generate_ini_file( is_harmonic=False, ): """Generate a CosmoSIS INI configuration file from template with modifications.""" - template_path = f"cosmosis_config/{template_base}" - output_path = f"cosmosis_config/cosmosis_pipeline_{args.cosmosis_root}{suffix}.ini" + template_path = Path("cosmosis_config") / template_base + output_path = Path(args.output_config_dir) / f"cosmosis_pipeline_{args.config_name_base}{suffix}.ini" with open(template_path, "r") as f: config_content = f.read() modifications = [] - relative_fits_file = f"data/{args.cosmosis_root}/cosmosis_{args.cosmosis_root}.fits" + relative_fits_file = args.config_relative_fits default_section = ( f"[DEFAULT]\nSCRATCH = {args.data_dir}\nFITS_FILE = {relative_fits_file}" ) @@ -396,6 +397,8 @@ def _generate_ini_file( pattern, replacement, config_content, flags=re.MULTILINE ) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: f.write(config_content) @@ -420,7 +423,7 @@ def generate_cosmosis_config(args): else: priors_file = "priors.ini" - os.makedirs("cosmosis_config", exist_ok=True) + os.makedirs(args.output_config_dir, exist_ok=True) _generate_ini_file( args, @@ -454,21 +457,40 @@ def parse_args(): description="Prepare CosmoSIS inference FITS files from real or mock data. " "Supports multiple xi input formats (separate files, combined FITS).", epilog=""" -Example for SP v1.4.6_A: - python cosmo_inference/scripts/cosmosis_fitting.py \\ - --cosmosis-root "SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ - --data-dir "/n09data/guerrini/output_chains" \\ +Example for SP_v1.4.6_leak_corr (real data): + python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmosis_fitting.py \\ + --cosmosis-root "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --data-dir "/n09data/guerrini/output_chains/SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ + --output-root "/home/guerrini/sp_validation/cosmo_inference" \\ + --xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_leak_corr_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked/covariance_SP_v1.4.6_leak_corr_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked_processed.txt" \\ + --use-rho-tau \\ + --rho-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-tau "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" \\ + --cl-file "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_SP_v1.4.6_leak_corr.fits" \\ + --cov-cl "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_cov_g_ng_iNKA_SP_v1.4.6_leak_corr.fits" + +Example for glass mock v0 (mock data): + python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmosis_fitting.py \\ + --mock \\ + --cosmosis-root "glass_mock_v0_00001" \\ + --data-dir "/n09data/guerrini/glass_mock_chains/glass_mock_v0_00001" \\ --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ - --out-file "cosmo_inference/data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --xi "notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - "notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --cov-xi "cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_processed.txt" \\ + --output-root "/home/guerrini/sp_validation/cosmo_inference" \\ + --output-basename "glass_mocks/v0/glass_mock_00001" \\ + --xi "/n09data/guerrini/glass_mock_v1.4.6/results/xi_glass_mock_00001_4096_nbins=20.fits" \\ + --cov-xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked_processed.txt" \\ --use-rho-tau \\ - --rho-stats "notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --tau-stats "notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ - --cov-tau "notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" + --rho-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "/n17data/cdaley/unions/pure_eb/results/glass_mock_rhotau_samples/00001/tau_stats_sampled.fits" \\ + --cov-tau "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" \\ + --cl-file "/n09data/guerrini/glass_mock_v1.4.6/results/cl_glass_mock_00001_4096.npy" \\ + --cov-cl "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_cov_g_ng_iNKA_SP_v1.4.6.fits" """, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( @@ -478,9 +500,6 @@ def parse_args(): "--data-dir", type=str, required=True, help="Output MCMC chain directory" ) parser.add_argument("--nz-file", type=str, required=True, help="Path to n(z) file") - parser.add_argument( - "--out-file", type=str, required=True, help="Path to output FITS file" - ) parser.add_argument( "--xi", nargs="+", @@ -490,7 +509,6 @@ def parse_args(): parser.add_argument( "--cov-xi", type=str, required=True, help="Xi covariance matrix file" ) - parser.add_argument( "--use-rho-tau", action="store_true", @@ -529,6 +547,24 @@ def parse_args(): parser.add_argument( "--mock", action="store_true", help="Mock data mode" ) + parser.add_argument( + "--output-root", + type=str, + required=True, + help=( + "Cosmo inference root directory (e.g., /home/guerrini/sp_validation/cosmo_inference). " + "FITS and config files are written under this root." + ), + ) + parser.add_argument( + "--output-basename", + type=str, + required=False, + help=( + "Optional override for the output sub-directory/basename inside --output-root. " + "Defaults to --cosmosis-root." + ), + ) return parser.parse_args() @@ -537,13 +573,31 @@ def parse_args(): args = parse_args() try: + output_basename = args.output_basename or args.cosmosis_root + output_root_path = Path(args.output_root).expanduser().resolve() + output_basename_path = Path(output_basename) + data_dir_root = output_root_path / "data" / output_basename_path + config_dir_root = output_root_path / "cosmosis_config" + data_dir_root.mkdir(parents=True, exist_ok=True) + config_dir_root.mkdir(parents=True, exist_ok=True) + out_file_path = data_dir_root / f"cosmosis_{args.cosmosis_root}.fits" + args.output_root = str(output_root_path) + args.output_config_dir = str(config_dir_root) + args.output_basename = str(output_basename_path) + args.config_name_base = str(output_basename_path).replace("/", "_") + args.config_relative_fits = str( + Path("data") / output_basename_path / f"cosmosis_{args.cosmosis_root}.fits" + ) + print("=" * 60) print("COSMOSIS_FITTING.PY") print("=" * 60) print(f"cosmosis_root: {args.cosmosis_root}") print(f"data_dir: {args.data_dir}") print(f"nz_file: {args.nz_file}") - print(f"out_file: {args.out_file}") + print(f"output_root: {args.output_root}") + print(f"output_basename: {args.output_basename}") + print(f"out_file: {out_file_path}") print(f"xi files: {args.xi}") print(f"cov_xi: {args.cov_xi}") print(f"use_rho_tau: {args.use_rho_tau}") @@ -568,9 +622,6 @@ def parse_args(): raise ValueError("--cl-file is required when --cov-cl is provided") os.makedirs(args.data_dir, exist_ok=True) - output_dir = os.path.dirname(args.out_file) - os.makedirs(output_dir, exist_ok=True) - print("Loading xi correlation functions...") if args.mock: # Mock mode expects a single combined FITS file @@ -638,8 +689,8 @@ def parse_args(): hdu_list.extend([tau_0_p_hdu, tau_2_p_hdu, rho_hdu]) hdul = fits.HDUList(hdu_list) - hdul.writeto(args.out_file, overwrite=True) - print(f"✓ FITS file written to {args.out_file}") + hdul.writeto(str(out_file_path), overwrite=True) + print(f"✓ FITS file written to {out_file_path}") print() generate_cosmosis_config(args) From f5abd15dbe3870444179b54ef1ce57beab390648 Mon Sep 17 00:00:00 2001 From: Sacha Guerrini Date: Fri, 7 Nov 2025 16:31:10 +0100 Subject: [PATCH 29/29] Merged harmonic and configuration space covariance to fix bug in inference run (Cosmosis). Added a prior file for harmonic space mocks. --- cosmo_inference/scripts/cosmosis_fitting.py | 116 ++++++++++++-------- 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 8ad2167..d221b9c 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -167,33 +167,6 @@ def cl_to_fits(ell, cl_ee, cl_bb): return cl_ee_hdu, cl_bb_hdu -def cov_cl_to_fits(cov_file, cov_hdu="COVAR_FULL"): - """Convert pseudo-C_ell covariance to a CosmoSIS ImageHDU.""" - if cov_file.endswith(".fits"): - with fits.open(cov_file) as hdul: - cov_data = np.asarray(hdul[cov_hdu].data, dtype=np.float64) - elif cov_file.endswith(".npy"): - cov_data = np.load(cov_file) - else: - raise NotImplementedError(f"Unsupported pseudo-Cl covariance format: {cov_file}") - - if cov_data.shape[0] != cov_data.shape[1]: - raise ValueError("Pseudo-Cl covariance matrix must be square") - - cov_hdu = fits.ImageHDU(cov_data, name="COVMAT_CELL") - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT_CELL", - "NAME_0": "CELL_EE", - "STRT_0": 0, - } - - for key, value in cov_dict.items(): - cov_hdu.header[key] = value - - return cov_hdu - - def tau_to_fits(filename, theta=None): """ Convert tau statistics to FITS format. @@ -238,12 +211,16 @@ def rho_to_fits(filename, theta=None): return rho_stat_hdu -def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): +def covdat_to_fits(filename_cov_xi, filename_cov_tau=None, filename_cov_cl=None, cov_cl_hdu=None): """ Convert CosmoCov covariance matrix to FITS format. If tau covariance provided, block with xi covariance. """ + if filename_cov_cl is not None: + assert cov_cl_hdu is not None, "Covariance HDU must be provided with cov_cl file." + + # Load xi covariance covmat_xi = np.loadtxt(filename_cov_xi) if filename_cov_tau is not None: @@ -259,6 +236,28 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): else: covmat = covmat_xi + covmat_real_space_dim = covmat.shape[0] + # Update the covariance with the pseudo-Cl part if provided + if filename_cov_cl is not None: + if filename_cov_cl.endswith(".fits"): + with fits.open(filename_cov_cl) as hdul: + covmat_cl = np.asarray(hdul[cov_cl_hdu].data, dtype=np.float64) + elif filename_cov_cl.endswith(".npy"): + covmat_cl = np.load(filename_cov_cl) + else: + raise NotImplementedError(f"Unsupported pseudo-Cl covariance format: {filename_cov_cl}") + + if covmat_cl.shape[0] != covmat_cl.shape[1]: + raise ValueError("Pseudo-Cl covariance matrix must be square") + + covmat = np.block( + [ + [covmat, np.zeros((len(covmat), len(covmat_cl)))], + [np.zeros((len(covmat_cl), len(covmat))), covmat_cl], + ] + ) + + if len(covmat) != len(covmat[0]): raise RuntimeError("Covariance matrix is not square") @@ -279,6 +278,17 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): "NAME_3": "TAU_2_PLUS", "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), }) + + if filename_cov_tau is None: + filename_cov_cl and cov_dict.update({ + "NAME_2": "CELL_EE", + "STRT_2": covmat_real_space_dim, + }) + else: + filename_cov_cl and cov_dict.update({ + "NAME_4": "CELL_EE", + "STRT_4": covmat_real_space_dim, + }) for key, value in cov_dict.items(): cov_hdu.header[key] = value @@ -379,7 +389,7 @@ def _generate_ini_file( "\ndata_sets=CELL_EE" ) - covmat_line = "covmat_name=COVMAT_CELL" if is_harmonic else "covmat_name=COVMAT" + covmat_line = "covmat_name=COVMAT" modifications.append((r"^\[2pt_like\]", like_section)) modifications.append((r"^covmat_name=.*", covmat_line)) @@ -436,11 +446,17 @@ def generate_cosmosis_config(args): if args.cl_file: template_base_harmonic = "cosmosis_pipeline_A_ia_cell.ini" + + values_file_cell = "values_ia.ini" + if args.mock: + priors_file_cell = "priors_mock_cell.ini" + else: + priors_file_cell = "priors.ini" _generate_ini_file( args, template_base_harmonic, - priors_file, - values_file, + priors_file_cell, + values_file_cell, suffix="_cell", is_harmonic=True, ) @@ -643,17 +659,6 @@ def parse_args(): nz_hdu = nz_to_fits(args.nz_file) print("Loaded n(z)") - cl_ee_hdu = None - cl_bb_hdu = None - cov_cl_hdu = None - if args.cl_file: - print("Loading Cl data...") - ell, cl_ee, cl_bb = load_pseudo_cl(args.cl_file) - print(f"Loaded Cl: {len(ell)} multipoles") - cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) - cov_cl_hdu = cov_cl_to_fits(args.cov_cl, cov_hdu="COVAR_FULL") - print("Loaded pseudo-Cl covariance") - rho_hdu = None tau_0_p_hdu = None tau_2_p_hdu = None @@ -673,17 +678,38 @@ def parse_args(): cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=args.cov_tau) + cl_ee_hdu = None + cl_bb_hdu = None + if args.cl_file: + print("Loading Cl data...") + ell, cl_ee, cl_bb = load_pseudo_cl(args.cl_file) + print(f"Loaded Cl: {len(ell)} multipoles") + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) + if args.use_rho_tau: + cov_hdu = covdat_to_fits( + args.cov_xi, + filename_cov_tau=args.cov_tau, + filename_cov_cl=args.cov_cl, + cov_cl_hdu="COVAR_FULL" + ) + else: + cov_hdu = covdat_to_fits( + args.cov_xi, + filename_cov_tau=None, + filename_cov_cl=args.cov_cl, + cov_cl_hdu="COVAR_FULL" + ) + print("Loaded pseudo-Cl covariance") + pri_hdr = fits.Header() pri_hdu = fits.PrimaryHDU(header=pri_hdr) print("Assembling FITS file...") hdu_list = [pri_hdu, cov_hdu] - if cov_cl_hdu is not None: - hdu_list.append(cov_cl_hdu) hdu_list.extend([nz_hdu, xip_hdu, xim_hdu]) if args.cl_file: - hdu_list.extend([cl_ee_hdu, cl_bb_hdu]) + hdu_list.extend([cl_ee_hdu]) if args.use_rho_tau: hdu_list.extend([tau_0_p_hdu, tau_2_p_hdu, rho_hdu])