Read ShapePipe v1 and v2 products side by side

.gitignore

@@ -206,3 +206,6 @@ report.md
 check2.txt
 format2.txt
 residual.md
+
+# Generated by docs/source/_ext/catalogue_columns.py from config/columns/
+docs/source/catalogue_columns.md

CLAUDE.md

@@ -43,7 +43,8 @@ is the container (full scientific stack pre-built). For a local dev environment:
 - `cosmo_val.py`: Cosmology validation routines
 - `cosmology.py`: Cosmological calculations and theory
 - `galaxy.py`: Galaxy-specific processing
-- `io.py`: Input/output utilities
+- `grammar.py`: Column grammars: ShapePipe v1 -> v2 adapter and `column_map` renames (`adapt`)
+- `io.py`: Input/output; the catalogue reader (`read_catalogue`, `Catalogue`), which detects FITS/HDF5 layouts
 - `plots.py`: Plotting functions
 - `rho_tau.py`: Rho and tau statistics calculations
 - `statistics.py`: Cosmology-independent statistics (jackknife resampling, χ²/PTE, covariance↔correlation, OneCovariance reshaping)
@@ -67,10 +68,10 @@ snakemake --profile workflow/profiles/candide -s workflow/Snakefile \
 
 ### Configuration
 Main configuration in `scripts/calibration/params.py` with parameters:
-- `name`: Field/patch identifier
+- `campaign`: Campaign name (the ShapePipe tile list); names the input products
 - `data_dir`: Input data directory
 - `galaxy_cat_path`: Galaxy catalogue path (.fits/.hdf5)
-- `star_cat_path`: Star catalogue path (.fits)
+- `star_cat_path`: Star catalogue path (.hdf5, or a v1 .fits)
 
 ### Key Dependencies
 - astropy, numpy, scipy for core calculations

config/calibration/mask_from_compr_v1.X.11.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
 
   # Duplicate objects
@@ -24,29 +24,29 @@ dat:
     kind: greater_equal
     value: 2
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False

config/calibration/mask_from_compr_v1.X.6.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
   # Duplicate objects
   - col_name: overlap
@@ -23,29 +23,29 @@ dat:
     kind: greater_equal
     value: 2
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False

config/calibration/mask_v1.X.10.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
   # SExtractor flags
   - col_name: FLAGS
@@ -57,29 +57,29 @@ dat:
     kind: not_equal
     value: -10
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False

config/calibration/mask_v1.X.11.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
   # SExtractor flags
   - col_name: FLAGS
@@ -57,29 +57,29 @@ dat:
     kind: not_equal
     value: -10
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False

config/calibration/mask_v1.X.2.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
   # SExtractor flags
   - col_name: FLAGS
@@ -57,40 +57,40 @@ dat:
     kind: not_equal
     value: -10
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
   # Faint star halos
-  - col_name: 1_Faint_star_halos
+  - col_name: MASK_1_Faint_star_halos
     label: "Faint star halos"
     kind: equal
     value: False
 
   # Bright star halos
-  - col_name: 2_Bright_star_halos
+  - col_name: MASK_2_Bright_star_halos
     label: "Bright star halos"
     kind: equal
     value: False
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False

config/calibration/mask_v1.X.3.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
   # SExtractor flags
   - col_name: FLAGS
@@ -57,29 +57,29 @@ dat:
     kind: not_equal
     value: -10
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False

config/calibration/mask_v1.X.4.yaml

@@ -9,7 +9,7 @@ params:
   verbose: True
 
 # Masks
-## Using columns in 'dat' group (ShapePipe flags)
+## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar)
 dat:
   # SExtractor flags
   - col_name: FLAGS
@@ -57,40 +57,40 @@ dat:
     kind: not_equal
     value: -10
 
-## Using columns in 'dat_ext' group (post-processing flags)
-dat_ext:
+  # Healsparse mask bits and post-processing columns (the comprehensive
+  # HDF5's data_ext dataset)
   # Faint star halos
-  - col_name: 1_Faint_star_halos
+  - col_name: MASK_1_Faint_star_halos
     label: "Faint star halos"
     kind: equal
     value: False
 
   # Bright star halos
-  - col_name: 2_Bright_star_halos
+  - col_name: MASK_2_Bright_star_halos
     label: "Bright star halos"
     kind: equal
     value: False
 
   # Stars
-  - col_name: 4_Stars
+  - col_name: MASK_4_Stars
     label: "Stars"
     kind: equal
     value: False
 
   # Manual mask
-  - col_name: 8_Manual
+  - col_name: MASK_8_Manual
     label: "manual mask"
     kind: equal
     value: False
 
   # r-band footprint
-  - col_name: 64_r
+  - col_name: MASK_64_r
     label: "r-band imaging"
     kind: equal
     value: False
 
   # Maximask
-  - col_name: 1024_Maximask
+  - col_name: MASK_1024_Maximask
     label: "maximask"
     kind: equal
     value: False