diff --git a/.gitignore b/.gitignore index 9c07b05b..e9d09406 100644 --- a/.gitignore +++ b/.gitignore @@ -206,3 +206,6 @@ report.md check2.txt format2.txt residual.md + +# Generated by docs/source/_ext/catalogue_columns.py from config/columns/ +docs/source/catalogue_columns.md diff --git a/CLAUDE.md b/CLAUDE.md index b4447758..9ae25ba6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,8 @@ is the container (full scientific stack pre-built). For a local dev environment: - `cosmo_val.py`: Cosmology validation routines - `cosmology.py`: Cosmological calculations and theory - `galaxy.py`: Galaxy-specific processing -- `io.py`: Input/output utilities +- `grammar.py`: Column grammars: ShapePipe v1 -> v2 adapter and `column_map` renames (`adapt`) +- `io.py`: Input/output; the catalogue reader (`read_catalogue`, `Catalogue`), which detects FITS/HDF5 layouts - `plots.py`: Plotting functions - `rho_tau.py`: Rho and tau statistics calculations - `statistics.py`: Cosmology-independent statistics (jackknife resampling, χ²/PTE, covariance↔correlation, OneCovariance reshaping) @@ -67,10 +68,10 @@ snakemake --profile workflow/profiles/candide -s workflow/Snakefile \ ### Configuration Main configuration in `scripts/calibration/params.py` with parameters: -- `name`: Field/patch identifier +- `campaign`: Campaign name (the ShapePipe tile list); names the input products - `data_dir`: Input data directory - `galaxy_cat_path`: Galaxy catalogue path (.fits/.hdf5) -- `star_cat_path`: Star catalogue path (.fits) +- `star_cat_path`: Star catalogue path (.hdf5, or a v1 .fits) ### Key Dependencies - astropy, numpy, scipy for core calculations diff --git a/config/calibration/mask_from_compr_v1.X.11.yaml b/config/calibration/mask_from_compr_v1.X.11.yaml index 3fa9ee3e..d76d7fb3 100644 --- a/config/calibration/mask_from_compr_v1.X.11.yaml +++ b/config/calibration/mask_from_compr_v1.X.11.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # Duplicate objects @@ -24,29 +24,29 @@ dat: kind: greater_equal value: 2 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_from_compr_v1.X.6.yaml b/config/calibration/mask_from_compr_v1.X.6.yaml index 1c9b08e5..a71f9974 100644 --- a/config/calibration/mask_from_compr_v1.X.6.yaml +++ b/config/calibration/mask_from_compr_v1.X.6.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # Duplicate objects - col_name: overlap @@ -23,29 +23,29 @@ dat: kind: greater_equal value: 2 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 703ea91b..2678a49b 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.11.yaml b/config/calibration/mask_v1.X.11.yaml index 88e01643..b87aa267 100644 --- a/config/calibration/mask_v1.X.11.yaml +++ b/config/calibration/mask_v1.X.11.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index 846f14ff..e7ad7758 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,40 +57,40 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Faint star halos - - col_name: 1_Faint_star_halos + - col_name: MASK_1_Faint_star_halos label: "Faint star halos" kind: equal value: False # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index bcbbc620..3c14c5fd 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 903b9098..37b67325 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,40 +57,40 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Faint star halos - - col_name: 1_Faint_star_halos + - col_name: MASK_1_Faint_star_halos label: "Faint star halos" kind: equal value: False # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.4_im_sim.yaml b/config/calibration/mask_v1.X.4_im_sim.yaml index 794566d4..8f3636c2 100644 --- a/config/calibration/mask_v1.X.4_im_sim.yaml +++ b/config/calibration/mask_v1.X.4_im_sim.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -30,17 +30,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index 5f56b135..bf717380 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index 1b5f7da6..fc021a2c 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.6_ppv1.yaml b/config/calibration/mask_v1.X.6_ppv1.yaml index a3d29cda..bcaffb15 100644 --- a/config/calibration/mask_v1.X.6_ppv1.yaml +++ b/config/calibration/mask_v1.X.6_ppv1.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 @@ -84,29 +84,29 @@ dat: col_name2: MAG_GAAP_0p7_z2 value: -99 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index cbc802be..585481d9 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,34 +57,34 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index 58234d64..5dafbb72 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,40 +57,40 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Faint star halos - - col_name: 1_Faint_star_halos + - col_name: MASK_1_Faint_star_halos label: "Faint star halos" kind: equal value: False # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.9.yaml b/config/calibration/mask_v1.X.9.yaml index 4a448ce7..906578df 100644 --- a/config/calibration/mask_v1.X.9.yaml +++ b/config/calibration/mask_v1.X.9.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml index 94c5b9ae..9c15edab 100644 --- a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml +++ b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml @@ -45,36 +45,36 @@ ops: with: |2 # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components) - # --- dat_ext (post-processing / coverage masks) ------------------------ + # --- mask bits (post-processing / coverage masks) ---------------------- - why: >- - No coverage masks on sims: the whole dat_ext group (Stars, manual mask, - r-band footprint, Maximask) is survey post-processing with no analogue - in the simulated tiles. + No coverage masks on sims: the healsparse mask-bit cuts (stars, manual + mask, r coverage, Maximask) are survey post-processing with no analogue in the + simulated tiles. drop: |2 - ## Using columns in 'dat_ext' group (post-processing flags) - dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.9_im_sim.yaml b/config/calibration/mask_v1.X.9_im_sim.yaml index 1c6ff672..3c28cc03 100644 --- a/config/calibration/mask_v1.X.9_im_sim.yaml +++ b/config/calibration/mask_v1.X.9_im_sim.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS diff --git a/config/calibration/mask_v2.0.yaml b/config/calibration/mask_v2.0.yaml new file mode 100644 index 00000000..930ae42f --- /dev/null +++ b/config/calibration/mask_v2.0.yaml @@ -0,0 +1,160 @@ +# Config file for masking and calibration, ShapePipe v2 catalogues. +# +# ShapePipe v2 replaces the single IMAFLAGS_ISO column with per-reason boolean +# mask columns MASK__