diff --git a/.gitignore b/.gitignore index 19b1503..3208c72 100644 --- a/.gitignore +++ b/.gitignore @@ -171,5 +171,15 @@ notebooks/demo_calibrate_minimal_cat.ipynb notebooks/leakage_minimal.ipynb notebooks/demo_add_bands.ipynb -cosmo_inference/cosmosis_config/cosmosis_pipeline_S*.ini -cosmo_inference/cosmosis_config/priors_S*.ini \ No newline at end of file +CLAUDE.md +AGENTS.md +cosmo_inference/cosmosis_config/*.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_cell.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A_psf.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_A.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini +!cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini +!cosmo_inference/cosmosis_config/priors_mock.ini diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 4da2aaa..f729285 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -7,20 +7,42 @@ This folder contains the files neccessary to run the cosmological inference pipe To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) and [CosmoCov](https://github.com/CosmoLike/CosmoCov). To PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. ### To Run -Run the bash script within this folder +The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: +```bash +snakemake --profile slurm -j inference_fiducial ``` -$ ./pipeline.sh -``` -with one of the following flags: - -`--pcf`: This step runs the `cosmo_val.py` script to calculate the various 2 point correlation functions. It will also write the $\xi_{pm}$ correlation functions in a fits file. -`--covmat`: The covariance matrix is calculated here using CosmoCov, by reading in the `./cosmocov_config/cosmocov_{output_root}.ini` file. **Hence make sure the `output_root` here corresponds to the one entered in the prompt**. +This will automatically execute all steps: +1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` +2. Compute covariance matrices using CosmoCov +3. Prepare CosmoSIS data (FITS) via `cosmosis_fitting.py` +4. Run CosmoSIS inference + +For standalone FITS data preparation (real-space inputs plus optional pseudo-$C_\ell$ data), you can also use the Python script directly: + +```bash +python scripts/cosmosis_fitting.py \ + --cosmosis-root "catalog_version_config" \ + --data-dir "/path/to/output/chains" \ + --nz-file "/path/to/nz_file.txt" \ + --out-file "/path/to/output.fits" \ + --xi "/path/to/xi_plus.fits" "/path/to/xi_minus.fits" \ + --cov-xi "/path/to/covariance.txt" \ + --use-rho-tau \ + --rho-stats "/path/to/rho_stats.fits" \ + --tau-stats "/path/to/tau_stats.fits" \ + --cov-tau "/path/to/cov_tau.npy" \ + --cl-file "/path/to/pseudo_cl.fits" \ + --cov-cl "/path/to/pseudo_cl_cov.fits" +``` -`--inference`: This step writes out the relevant `./cosmosis_config/cosmosis_{output_root}.ini` file, in order to run CosmoSIS to conduct the cosmological inference. It also combines the data needed by CosmoSIS: the $\xi_{pm}$ fits files calculated in `cosmo_val.py`, the covariance matrix, and the nz catalogue, into a single `.fits` file. It also fetches the rho-statistics computed in `cosmo_val.py` to marginalize on PSF leakage parameters. To do so, specify the path to the `cosmo_val` outputs. Note that the Rho- and Tau-statistics have to be stored in a folder `rho_tau_stats` in this folder. If the 2PCF $\xi_{pm}$ are not found it will raise an `Error` to run `cosmo_val`. If the Rho- and Tau-statistics do not exist, the script will raise a `Warning` to run `cosmo_val` but will create a data vector without the Rho- and Tau-statistics. +You can view all available options with: +```bash +python scripts/cosmosis_fitting.py --help +``` -`--mcmc_process`: You can finally analyse the chains with the `MCMC.ipynb` notebook. +Ensure the pseudo-$C_\ell$ spectra (`pseudo_cl_*.fits`) and their covariance (`pseudo_cl_cov_*.fits`) produced by `cosmo_val.py` exist for the requested catalog version (or mock seed) before running the standalone command. -This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. \ No newline at end of file +This is the pipeline used to derive cosmological constraints with cosmic shear data from the UNIONS v1.4 catalogue. diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 0000000..6be956e --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini new file mode 100644 index 0000000..6bc3a7d --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +FITS_FILE = data/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/cosmosis_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.fits +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +priors = cosmosis_config/priors_psf.ini +values = cosmosis_config/values_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/test_new_pipeline + +[polychord] +polychord_outfile_root = SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1/samples_SP_v1.4.5_leak_corr_A_minsep=1_maxsep=250_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +add_xi_sys=T +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini new file mode 100644 index 0000000..d64ac64 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini new file mode 100644 index 0000000..34a7209 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/output_chains/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1 +FITS_FILE = data/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/cosmosis_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_psf.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell + + +[polychord] +polychord_outfile_root = SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1/samples_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT_CELL +cut_zeros=F +like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini new file mode 100644 index 0000000..85374a2 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001.ini @@ -0,0 +1,124 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/glass_mock_chains/glass_mock_00001 +FITS_FILE = data/glass_mock_00001/cosmosis_glass_mock_00001.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_mock.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias add_xi_sys tau_from_rho 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/glass_mock_00001 + +[polychord] +polychord_outfile_root = glass_mock_00001 +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/glass_mock_00001/samples_glass_mock_00001.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[add_xi_sys] +file = %(COSMOSIS_DIR)s/shear/xi_sys/xi_sys_psf.py +data_file=%(FITS_FILE)s +rho_stats_name=RHO_STATS + +[tau_from_rho] +file = %(COSMOSIS_DIR)s/shear/xi_sys/tau_from_rho.py +data_file=%(FITS_FILE)s + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py +data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS +add_xi_sys=T +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +like_name=2pt_like +angle_range_XI_PLUS_1_1= 3.0 150.0 +angle_range_XI_MINUS_1_1= 10.0 200.0 \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini new file mode 100644 index 0000000..f1a03fd --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_glass_mock_00001_cell.ini @@ -0,0 +1,113 @@ +#parameters used elsewhere in this file +[DEFAULT] +SCRATCH = /n09data/guerrini/glass_mock_chains/glass_mock_00001 +FITS_FILE = data/glass_mock_00001/cosmosis_glass_mock_00001.fits +COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library + + +[pipeline] +values = cosmosis_config/values_psf.ini +priors = cosmosis_config/priors_mock.ini +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[test] +save_dir = %(SCRATCH)s/best_fit/glass_mock_00001_cell + + +[polychord] +polychord_outfile_root = glass_mock_00001_cell +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[output] +filename = %(SCRATCH)s/glass_mock_00001/samples_glass_mock_00001_cell.txt +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_cl +verbose = F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_sets=CELL_EE +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT_CELL +cut_zeros=F +like_name=2pt_like \ No newline at end of file diff --git a/cosmo_inference/cosmosis_config/priors_mock.ini b/cosmo_inference/cosmosis_config/priors_mock.ini new file mode 100644 index 0000000..0fe8933 --- /dev/null +++ b/cosmo_inference/cosmosis_config/priors_mock.ini @@ -0,0 +1,11 @@ +[intrinsic_alignment_parameters] +A = gaussian 0.83 0.7 + +[cosmological_parameters] +ombh2 = gaussian 0.0244 0.00038 + +[shear_calibration_parameters] +m1 = gaussian 0.0 0.01 + +[nofz_shifts] +bias_1 = gaussian -0.033 0.013 diff --git a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py index e0b371c..5d7b740 100644 --- a/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py +++ b/cosmo_inference/notebooks/2D_cosmic_shear_paper_plots/workflow/scripts/eb_plots.py @@ -48,7 +48,6 @@ cv = CosmologyValidation( versions=[config["version"]], - data_base_dir="/n17data/mkilbing/astro/data", ) # %% eb_results = cv.calculate_pure_eb(version=config["version"], **config["pure_eb"]) diff --git a/cosmo_inference/pipeline.sh b/cosmo_inference/pipeline.sh deleted file mode 100755 index 8d20cb9..0000000 --- a/cosmo_inference/pipeline.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# Transform long options to short ones -for arg in "$@"; do - shift - case "$arg" in - '--help') set -- "$@" '-h' ;; - '--pcf') set -- "$@" '-p' ;; - '--covmat') set -- "$@" '-c' ;; - '--inference') set -- "$@" '-i' ;; - '--mcmc_process') set -- "$@" '-m' ;; - *) set -- "$@" "$arg" ;; - esac -done - -# Parse short options -OPTIND=1 -while getopts "hpcim" opt -do - case "$opt" in - 'h') - echo "Please input a flag: --help, --pcf, --covmat, --inference or --mcmc_process "; - exit 0 - ;; - 'p') - echo "Running cosmo_val.py to calculate 2 point correlation functions"; - python notebooks/cosmo_val/cosmo_val.py - ;; - 'c') - read -p 'COVARIANCE FILE: ' covmat_file; - read -p 'OUTPUT STUB (without extension): ' output_stub; - echo "Processing covariance matrix"; - python scripts/cosmocov_process.py $covmat_file $output_stub - ;; - 'i') - read -p 'COSMOSIS ROOT: ' cosmosis_root; - - # Create the folder if it does not exist - if [ ! -d "data/$cosmosis_root" ]; then - mkdir -p "data/$cosmosis_root" - echo "Directory 'data/$cosmosis_root' created." - else - echo "Directory 'data/$cosmosis_root' already exists." - fi - - read -p 'COSMO_VAL OUTPUT FOLDER: ' output_folder; - read -p 'NZ FILE:' nz_file; - read -p 'OUTPUT MCMC CHAIN FOLDER: ' data; - read -p 'USE PSEUDO_CELL? (y/n): ' pseudo_cell; - - if [ "${pseudo_cell}" == "y" ]; then - - read -p 'CL ROOT: ' cl_root; - read -p 'GAUSSIAN COVARIANCE? (iNKA/OneCovariance): ' gaussian_part; - - - echo "Using pseudo cell" - out_file="data/${cosmosis_root}/cosmosis_${cosmosis_root}_cell.fits" - - python scripts/cosmosis_fitting.py $cosmosis_root $output_folder $nz_file $pseudo_cell $out_file $cl_root $gaussian_part; - else - read -p 'XI ROOT: ' xi_root; - read -p 'USE RHO/TAU_STATS? (y/n): ' rhotau_stats; - echo $rhotau_stats - read -p 'TAU ROOT: ' tau_root; - read -p 'COV_XI MAT TXT FILE:' covmat; - - out_file="data/${cosmosis_root}/cosmosis_${cosmosis_root}.fits"; - - #LG: add check if xi_plus/xi_minus fits file exists - python scripts/cosmosis_fitting.py $cosmosis_root $output_folder $nz_file $pseudo_cell $out_file $xi_root $rhotau_stats $tau_root $covmat; - - fi - - if [ "${pseudo_cell}" == "y" ]; then - output_ini_file="cosmosis_config/cosmosis_pipeline_${cosmosis_root}_cell.ini" - cp cosmosis_config/cosmosis_pipeline_A_ia_cell.ini $output_ini_file - else - output_ini_file="cosmosis_config/cosmosis_pipeline_${cosmosis_root}.ini" - if [ "${rhotau_stats}" == "y" ]; then - cp cosmosis_config/cosmosis_pipeline_A_psf.ini $output_ini_file; - else - cp cosmosis_config/cosmosis_pipeline_A_ia.ini $output_ini_file; - fi - fi - - sed -i "/^\[DEFAULT\]/a\SCRATCH = ${data}" $output_ini_file; - sed -i "/^\[DEFAULT\]/a\FITS_FILE = ${out_file}" $output_ini_file; - if [ "${pseudo_cell}" == "y" ]; then - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${cosmosis_root}_cell/samples_${cosmosis_root}_cell.txt" $output_ini_file; - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=CELL_EE" $output_ini_file; - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${cosmosis_root}_cell" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${cosmosis_root}_cell" $output_ini_file; - else - sed -i "/^\[output\]/a\filename = %(SCRATCH)s/${cosmosis_root}/samples_${cosmosis_root}.txt" $output_ini_file; - if [ "${rhotau_stats}" == "y" ]; then - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_psf.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors_psf.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS" $output_ini_file; - sed -i "/^\[2pt_like]/a\add_xi_sys=T" $output_ini_file; - else - sed -i "/^\[pipeline\]/a\values = cosmosis_config/values_ia.ini" $output_ini_file; - sed -i "/^\[pipeline\]/a\priors = cosmosis_config/priors.ini" $output_ini_file; - sed -i "/^\[2pt_like]/a\file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" $output_ini_file; - sed -i "/^\[2pt_like]/a\data_sets=XI_PLUS XI_MINUS" $output_ini_file; - fi - sed -i "/^\[polychord\]/a\polychord_outfile_root = ${root}" $output_ini_file; - sed -i "/^\[test\]/a\save_dir = %(SCRATCH)s/best_fit/${root}" $output_ini_file; - fi - echo "Prepared CosmoSIS configuration file in $output_ini_file"; - echo "You can now run the inference with the command: cosmosis $output_ini_file" - ;; - 'm') - # LG: also convert this into a script to directly output contour plots - echo "Run the cosmo_inference/notebooks/MCMC.ipynb notebook to analyse your chains" - ;; - '?') - print_usage >&2; - exit 1 - ;; - esac -done -shift $(expr $OPTIND - 1) # remove options from positional parameters \ No newline at end of file diff --git a/cosmo_inference/scripts/cosmosis_fitting.py b/cosmo_inference/scripts/cosmosis_fitting.py index 7f4d918..d221b9c 100644 --- a/cosmo_inference/scripts/cosmosis_fitting.py +++ b/cosmo_inference/scripts/cosmosis_fitting.py @@ -1,142 +1,228 @@ #!/usr/bin/env python # coding: utf-8 +import argparse import os +import re import sys from pathlib import Path -import matplotlib.pylab as plt import numpy as np from astropy.io import fits +# ============================================================================ +# SECTION 1: FITS FORMAT CONVERTERS (data → FITS HDU) +# ============================================================================ + + +def _create_2pt_hdu(values, theta, name, quant1, quant2): + """Create standardized 2-point correlation FITS HDU.""" + nbins = len(values) + lst = np.arange(1, nbins + 1) + + col1 = fits.Column(name="BIN1", format="K", array=np.ones(nbins)) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(nbins)) + col3 = fits.Column(name="ANGBIN", format="K", array=lst) + col4 = fits.Column(name="VALUE", format="D", array=values) + col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=theta) + + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + hdu = fits.BinTableHDU.from_columns(coldefs, name=name) + + hdu_dict = { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + } + for key, value in hdu_dict.items(): + hdu.header[key] = value + + return hdu + + +def nz_to_fits(filename): + """Convert n(z) text file to FITS format.""" + line = np.loadtxt(filename, max_rows=1) + nbins = len(line) - 1 + + z_low = np.loadtxt(filename, usecols=0) + nstep = z_low[1] - z_low[0] + + z_mid = z_low + nstep / 2 + z_high = np.append(z_low[1:], z_low[-1] + nstep) + + col1 = fits.Column(name="Z_LOW", format="D", array=z_low) + col2 = fits.Column(name="Z_MID", format="D", array=z_mid) + col3 = fits.Column(name="Z_HIGH", format="D", array=z_high) + cols = [col1, col2, col3] + + for i in range(nbins): + bin_col = np.loadtxt(filename, usecols=i + 1) + hdu_col = fits.Column(name="BIN%d" % (i + 1), format="D", array=bin_col) + cols.append(hdu_col) + + coldefs = fits.ColDefs(cols) + nz_hdu = fits.BinTableHDU.from_columns(coldefs, name="NZDATA") + + nz_lens_dict = { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": nbins, + "NZ": len(z_low), + } + + for key, value in nz_lens_dict.items(): + nz_hdu.header[key] = value + + return nz_hdu -# transforms treecorr fits file of correlation functions into CosmoSIS-friendly 2pt FITS extension to be read by 2pt_likelihood -def treecorr_to_fits(filename1, filename2): +def treecorr_to_fits(filename1, filename2): + """Load xi+ and xi- from separate TreeCorr FITS files.""" xiplus_hdu = fits.open(filename1) ximinus_hdu = fits.open(filename2) - return xiplus_hdu[1], ximinus_hdu[1] -def tau_to_fits(filename, theta=None): - """ - Convert tau statistics to CosmoSIS FITS format. - - Parameters: - filename : str - Path to tau statistics FITS file - theta : array-like, optional - Angular separation values to use. If provided, overrides the theta values - from the tau statistics file. This is useful for forcing consistency with - xi correlation function angular separations. - """ - tau_stats = fits.getdata(filename) +def parse_combined_xi_fits(filepath): + """Parse combined FITS file with xip/xim columns (mock format).""" + with fits.open(filepath) as hdul: + xi_data = hdul[1].data + xip_vals = xi_data["xip"] + xim_vals = xi_data["xim"] + xi_theta = xi_data["ANG"] if "ANG" in xi_data.names else xi_data["meanr"] - # Use provided theta if given, otherwise use tau's original theta values - if theta is not None: - ang = theta - print(f"Using provided theta values for tau statistics (forcing consistency)") - else: - ang = tau_stats["theta"] - print(f"Using original tau theta values") - - nbins = len(ang) + xip_hdu = _create_2pt_hdu(xip_vals, xi_theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = _create_2pt_hdu(xim_vals, xi_theta, "XI_MINUS", "G-R", "G-R") + + return xip_hdu, xim_hdu + + +def load_pseudo_cl(cl_file): + """Load pseudo-C_ell spectra from .npy or FITS formats.""" + if cl_file.endswith(".npy"): + cl_block = np.load(cl_file) + if cl_block.shape[0] < 5: + raise ValueError( + f"Unexpected C_ell array shape {cl_block.shape} for {cl_file}" + ) + ell = np.asarray(cl_block[0], dtype=np.float64) + cl_ee = np.asarray(cl_block[1], dtype=np.float64) + cl_bb = np.asarray(cl_block[4], dtype=np.float64) + return ell, cl_ee, cl_bb + + if cl_file.endswith(".fits"): + with fits.open(cl_file) as hdul: + data = hdul[1].data + ell = np.asarray(data["ELL"], dtype=np.float64) + cl_ee = np.asarray(data["EE"], dtype=np.float64) + cl_bb = np.asarray(data["BB"], dtype=np.float64) + return ell, cl_ee, cl_bb + + raise NotImplementedError(f"Cl format not supported: {cl_file}") + + +def cl_to_fits(ell, cl_ee, cl_bb): + """Convert C_ell arrays to CosmoSIS FITS HDUs for EE and BB.""" + nbins = len(ell) lst = np.arange(1, nbins + 1) - # Create fits HDU for tau_0_+ - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col1 = fits.Column(name="BIN1", format="K", array=np.ones(nbins)) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(nbins)) col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=tau_stats["tau_0_p"]) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=ang) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - tau_0_p_hdu = fits.BinTableHDU.from_columns(coldefs, name="TAU_0_PLUS") + col5 = fits.Column(name="ANG", format="D", array=ell) - # Create fits HDU for tau_2_+ - col4 = fits.Column(name="VALUE", format="D", array=tau_stats["tau_2_p"]) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - tau_2_p_hdu = fits.BinTableHDU.from_columns(coldefs, name="TAU_2_PLUS") + col4_ee = fits.Column(name="VALUE", format="D", array=cl_ee) + coldefs_ee = fits.ColDefs([col1, col2, col3, col4_ee, col5]) + cl_ee_hdu = fits.BinTableHDU.from_columns(coldefs_ee, name="CELL_EE") - # Append tau_0_p/tau_2_p header info - tau_0_p_dict = { + cl_ee_dict = { "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "P+R", + "QUANT1": "GEF", + "QUANT2": "GEF", "KERNEL_1": "NZ_SOURCE", "KERNEL_2": "NZ_SOURCE", "WINDOWS": "SAMPLE", } + for key, value in cl_ee_dict.items(): + cl_ee_hdu.header[key] = value - for key in tau_0_p_dict: - tau_0_p_hdu.header[key] = tau_0_p_dict[key] + col4_bb = fits.Column(name="VALUE", format="D", array=cl_bb) + coldefs_bb = fits.ColDefs([col1, col2, col3, col4_bb, col5]) + cl_bb_hdu = fits.BinTableHDU.from_columns(coldefs_bb, name="CELL_BB") - tau_2_p_dict = { + cl_bb_dict = { "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "SR+R", + "QUANT1": "GBF", + "QUANT2": "GBF", "KERNEL_1": "NZ_SOURCE", "KERNEL_2": "NZ_SOURCE", "WINDOWS": "SAMPLE", } + for key, value in cl_bb_dict.items(): + cl_bb_hdu.header[key] = value - for key in tau_2_p_dict: - tau_2_p_hdu.header[key] = tau_2_p_dict[key] + return cl_ee_hdu, cl_bb_hdu - return tau_0_p_hdu, tau_2_p_hdu -def pseudo_cl_to_fits(filename): - pseudo_cl = fits.getdata(filename) - cl_ee = pseudo_cl["EE"] - ell = pseudo_cl["ELL"] +def tau_to_fits(filename, theta=None): + """ + Convert tau statistics to FITS format. + If theta provided, override original values for consistency with xi. + """ + tau_stats = fits.getdata(filename) + + if theta is not None: + ang = theta + print("Using provided theta values for tau statistics (forcing consistency)") + else: + ang = tau_stats["theta"] + print("Using original tau theta values") + + tau_0_p_hdu = _create_2pt_hdu(tau_stats["tau_0_p"], ang, "TAU_0_PLUS", "G+R", "P+R") + tau_2_p_hdu = _create_2pt_hdu( + tau_stats["tau_2_p"], ang, "TAU_2_PLUS", "G+R", "SR+R" + ) - nbins = len(ell) - lst = np.arange(1, nbins+1) - - #Create fits HDU for tau_0_+ - col1 = fits.Column(name ='BIN1', format ='K', array = np.ones(len(lst))) - col2 = fits.Column(name ='BIN2', format ='K', array = np.ones(len(lst))) - col3 = fits.Column(name ='ANGBIN', format ='K', array = lst) - col4 = fits.Column(name = 'VALUE', format = 'D', array = cl_ee) - col5 = fits.Column(name = 'ANG', format = 'D', array = ell) - coldefs = fits.ColDefs([col1,col2,col3,col4,col5]) - pseudo_cl_hdu = fits.BinTableHDU.from_columns(coldefs,name ='CELL_EE') - - pseudo_cl_dict = { - '2PTDATA': 'T', - 'QUANT1' : 'GEF', - 'QUANT2' : 'GEF', - 'KERNEL_1': 'NZ_SOURCE', - 'KERNEL_2': 'NZ_SOURCE', - 'WINDOWS': 'SAMPLE' - } + return tau_0_p_hdu, tau_2_p_hdu - for key in pseudo_cl_dict: - pseudo_cl_hdu.header[key] = pseudo_cl_dict[key] - return pseudo_cl_hdu +def rho_to_fits(filename, theta=None): + """ + Convert rho statistics to FITS format. + If theta provided, override original values for consistency with xi. + """ + rho_stat_hdul = fits.open(filename) + rho_stat_hdu = rho_stat_hdul[1].copy() + rho_stat_hdu.name = "RHO_STATS" -def cov_pseudo_cl_to_fits(filename): - cov_pseudo_cl = fits.open(filename) - cov_ee = cov_pseudo_cl["COVAR_FULL"].data + if theta is not None: + print( + "Forcing rho statistics to use provided theta values (forcing consistency)" + ) + rho_stat_hdu.data = rho_stat_hdu.data.copy() + rho_stat_hdu.data["theta"] = theta + else: + print("Using original rho theta values") - cov_hdu = fits.ImageHDU(cov_ee) - cov_dict = { - 'COVDATA': 'True', - 'EXTNAME': 'COVMAT', - 'NAME_0': 'CELL_EE', - 'STRT_0': 0, - } + rho_stat_hdul.close() + return rho_stat_hdu - for key in cov_dict: - cov_hdu.header[key] = cov_dict[key] - return cov_hdu +def covdat_to_fits(filename_cov_xi, filename_cov_tau=None, filename_cov_cl=None, cov_cl_hdu=None): + """ + Convert CosmoCov covariance matrix to FITS format. -#transforms text file of CosmoCov data into covmat HDU extension -def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): + If tau covariance provided, block with xi covariance. + """ + if filename_cov_cl is not None: + assert cov_cl_hdu is not None, "Covariance HDU must be provided with cov_cl file." - # read in cov txt data from CosmoCov + # Load xi covariance covmat_xi = np.loadtxt(filename_cov_xi) + if filename_cov_tau is not None: covmat_tau = np.load(filename_cov_tau) nbins = int(len(covmat_tau) / 3) @@ -147,282 +233,494 @@ def covdat_to_fits(filename_cov_xi, filename_cov_tau=None): [np.zeros((len(covmat_tau), len(covmat_xi))), covmat_tau], ] ) + else: + covmat = covmat_xi + + covmat_real_space_dim = covmat.shape[0] + # Update the covariance with the pseudo-Cl part if provided + if filename_cov_cl is not None: + if filename_cov_cl.endswith(".fits"): + with fits.open(filename_cov_cl) as hdul: + covmat_cl = np.asarray(hdul[cov_cl_hdu].data, dtype=np.float64) + elif filename_cov_cl.endswith(".npy"): + covmat_cl = np.load(filename_cov_cl) + else: + raise NotImplementedError(f"Unsupported pseudo-Cl covariance format: {filename_cov_cl}") + + if covmat_cl.shape[0] != covmat_cl.shape[1]: + raise ValueError("Pseudo-Cl covariance matrix must be square") + + covmat = np.block( + [ + [covmat, np.zeros((len(covmat), len(covmat_cl)))], + [np.zeros((len(covmat_cl), len(covmat))), covmat_cl], + ] + ) + if len(covmat) != len(covmat[0]): - print("Error: covmat not square!") - exit() + raise RuntimeError("Covariance matrix is not square") + + cov_hdu = fits.ImageHDU(covmat) + cov_dict = { + "COVDATA": "True", + "EXTNAME": "COVMAT", + "NAME_0": "XI_PLUS", + "STRT_0": 0, + "NAME_1": "XI_MINUS", + "STRT_1": int(len(covmat_xi) / 2), + } + + filename_cov_tau and cov_dict.update({ + "NAME_2": "TAU_0_PLUS", + "STRT_2": len(covmat_xi), + "NAME_3": "TAU_2_PLUS", + "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), + }) + + if filename_cov_tau is None: + filename_cov_cl and cov_dict.update({ + "NAME_2": "CELL_EE", + "STRT_2": covmat_real_space_dim, + }) else: - # create covmat ImageHDU - cov_hdu = fits.ImageHDU(covmat) - - # create header - if filename_cov_tau is None: - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT", - "NAME_0": "XI_PLUS", - "STRT_0": 0, - "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat) / 2), - } - else: - cov_dict = { - "COVDATA": "True", - "EXTNAME": "COVMAT", - "NAME_0": "XI_PLUS", - "STRT_0": 0, - "NAME_1": "XI_MINUS", - "STRT_1": int(len(covmat_xi) / 2), - "NAME_2": "TAU_0_PLUS", - "STRT_2": len(covmat_xi), - "NAME_3": "TAU_2_PLUS", - "STRT_3": len(covmat_xi) + int(len(covmat_tau) / 2), - } - for key in cov_dict: - cov_hdu.header[key] = cov_dict[key] + filename_cov_cl and cov_dict.update({ + "NAME_4": "CELL_EE", + "STRT_4": covmat_real_space_dim, + }) + + for key, value in cov_dict.items(): + cov_hdu.header[key] = value return cov_hdu -# transforms nz data (that was used in CosmoCov format) into nzdat HDU extension -def nz_to_fits(filename): +# ============================================================================ +# SECTION 2: VALIDATION & CONSISTENCY CHECKS +# ============================================================================ - line = np.loadtxt(filename, max_rows=1) - nbins = len(line) - 1 - z_low = np.loadtxt(filename, usecols=0) +def check_meanr_consistency(xi_theta, tau_theta, rho_theta, threshold=1.0): + """Check and report theta consistency across xi, tau, and rho statistics.""" + print("=" * 60) + print("MEANR CONSISTENCY CHECK") + print("=" * 60) - nstep = z_low[1] - z_low[0] + tau_diff = np.abs((tau_theta - xi_theta) / xi_theta) * 100 + rho_diff = np.abs((rho_theta - xi_theta) / xi_theta) * 100 - z_mid = z_low + nstep / 2 - z_high = np.append(z_low[1:], z_low[-1] + nstep) + print(f"Xi theta range: {xi_theta.min():.6f} - {xi_theta.max():.6f} arcmin") + print(f"Tau theta range: {tau_theta.min():.6f} - {tau_theta.max():.6f} arcmin") + print(f"Rho theta range: {rho_theta.min():.6f} - {rho_theta.max():.6f} arcmin") + print() + print(f"Max tau-xi relative difference: {tau_diff.max():.3f}%") + print(f"Mean tau-xi relative difference: {tau_diff.mean():.3f}%") + print(f"Max rho-xi relative difference: {rho_diff.max():.3f}%") + print(f"Mean rho-xi relative difference: {rho_diff.mean():.3f}%") + + if tau_diff.max() > threshold or rho_diff.max() > threshold: + print(f"\nWARNING: Meanr differences exceed {threshold}% threshold") + if tau_diff.max() > threshold: + print(f" Tau-xi: max {tau_diff.max():.3f}%") + if rho_diff.max() > threshold: + print(f" Rho-xi: max {rho_diff.max():.3f}%") + else: + print(f"✓ All differences below {threshold}% threshold") - # create hdu for histogram bin - col1 = fits.Column(name="Z_LOW", format="D", array=z_low) - col2 = fits.Column(name="Z_MID", format="D", array=z_mid) - col3 = fits.Column(name="Z_HIGH", format="D", array=z_high) - cols = [col1, col2, col3] + print("=" * 60) + print() - for i in range(nbins): - bin_col = np.loadtxt(filename, usecols=i + 1) - hdu_col = fits.Column(name="BIN%d" % (i + 1), format="D", array=bin_col) - cols.append(hdu_col) - coldefs = fits.ColDefs(cols) - nz_hdu = fits.BinTableHDU.from_columns(coldefs, name="NZDATA") +# ============================================================================ +# SECTION 3: COSMOSIS CONFIGURATION GENERATION +# ============================================================================ + + +def _generate_ini_file( + args, + template_base, + priors_file, + values_file, + suffix="", + is_harmonic=False, +): + """Generate a CosmoSIS INI configuration file from template with modifications.""" + template_path = Path("cosmosis_config") / template_base + output_path = Path(args.output_config_dir) / f"cosmosis_pipeline_{args.config_name_base}{suffix}.ini" + + with open(template_path, "r") as f: + config_content = f.read() + + modifications = [] + + relative_fits_file = args.config_relative_fits + default_section = ( + f"[DEFAULT]\nSCRATCH = {args.data_dir}\nFITS_FILE = {relative_fits_file}" + ) + modifications.append((r"^\[DEFAULT\]", default_section)) + + output_section = ( + f"[output]\nfilename = %(SCRATCH)s/{args.cosmosis_root}/samples_" + f"{args.cosmosis_root}{suffix}.txt" + ) + modifications.append((r"^\[output\]", output_section)) + + pipeline_section = ( + f"[pipeline]\nvalues = cosmosis_config/{values_file}\npriors = " + f"cosmosis_config/{priors_file}" + ) + modifications.append((r"^\[pipeline\]", pipeline_section)) + + if not is_harmonic: + if args.use_rho_tau: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like_xi_sys.py" + "\ndata_sets=XI_PLUS XI_MINUS TAU_0_PLUS TAU_2_PLUS\nadd_xi_sys=T" + ) + else: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" + "\ndata_sets=XI_PLUS XI_MINUS" + ) + else: + like_section = ( + "[2pt_like]\nfile = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py" + "\ndata_sets=CELL_EE" + ) - # create n(z) header - nz_lens_dict = { - "NZDATA": "T ", - "EXTNAME": "NZ_SOURCE", - "NBIN": nbins, - "NZ": len(z_low), - } + covmat_line = "covmat_name=COVMAT" - for key in nz_lens_dict: - nz_hdu.header[key] = nz_lens_dict[key] + modifications.append((r"^\[2pt_like\]", like_section)) + modifications.append((r"^covmat_name=.*", covmat_line)) - return nz_hdu + poly_section = f"[polychord]\npolychord_outfile_root = {args.cosmosis_root}{suffix}" + modifications.append((r"^\[polychord\]", poly_section)) + test_section = ( + f"[test]\nsave_dir = %(SCRATCH)s/best_fit/{args.cosmosis_root}{suffix}" + ) + modifications.append((r"^\[test\]", test_section)) -def rho_to_fits(filename, theta=None): - """ - Convert rho statistics to CosmoSIS FITS format. - - Parameters: - filename : str - Path to rho statistics FITS file - theta : array-like, optional - Angular separation values to use. If provided, replaces the theta values - in the rho statistics file. This is useful for forcing consistency with - xi correlation function angular separations. - """ - rho_stat_hdul = fits.open(filename) - rho_stat_hdu = rho_stat_hdul[1].copy() # Create a copy to avoid modifying the original - rho_stat_hdu.name = "RHO_STATS" - - # Force rho to use provided theta if given - if theta is not None: - print(f"Forcing rho statistics to use provided theta values (forcing consistency)") - # Update the theta column in the data - rho_stat_hdu.data = rho_stat_hdu.data.copy() # Make data writable - rho_stat_hdu.data['theta'] = theta - else: - print(f"Using original rho theta values") - - rho_stat_hdul.close() # Close the original file - return rho_stat_hdu - -if __name__ == "__main__": - -#combines all the data: 2pt correlation functions from treecorr, covmat from CosmoCov (must already be combined into 1 txt file), nz txt data -#into 1 fits file to be read by CosmoSIS 2pt-likelihood function -#give file path of each of the 3 components as input, also file path of desired output FITS file -#outputs nothing, but writes a new FITS file with appropriate extensions + for pattern, replacement in modifications: + config_content = re.sub( + pattern, replacement, config_content, flags=re.MULTILINE + ) - # Print all input arguments - print("=" * 60) - print("COSMOSIS_FITTING.PY - INPUT ARGUMENTS") - print("=" * 60) - print(f"Script name: {sys.argv[0]}") - print(f"Total arguments: {len(sys.argv)}") - - arg_names = [ - "script_name", - "cosmosis_root", - "output_folder", - "nz_file", - "use_pseudo_cell", - "out_file", - ] - - for i, arg in enumerate(sys.argv): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {arg}") - print("=" * 60) - print() + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + f.write(config_content) - cosmosis_root = sys.argv[1] - output_folder = sys.argv[2] - nz_file = sys.argv[3] - out_file = sys.argv[5] + config_type = "harmonic-space" if is_harmonic else "real-space" + print(f"Prepared CosmoSIS {config_type} configuration file in {output_path}") + print(f"You can now run inference with the command: cosmosis {output_path}") - use_pseudo_cell = sys.argv[4] - use_pseudo_cell = True if use_pseudo_cell == 'y' else False - if use_pseudo_cell: - arg_names += ["cl_root", "gaussian_part"] +def generate_cosmosis_config(args): + """Generate CosmoSIS INI files (real-space and optional harmonic-space).""" + if args.use_rho_tau: + template_base_realspace = "cosmosis_pipeline_A_psf.ini" + values_file = "values_psf.ini" + else: + template_base_realspace = "cosmosis_pipeline_A_ia.ini" + values_file = "values_ia.ini" - for i, argv in enumerate(sys.argv[6:], start=6): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {argv}") - else: - print(f" argv[{i}] (unexpected): {argv}") - print("=" * 60) - print() + if args.mock: + priors_file = "priors_mock.ini" + elif args.use_rho_tau: + priors_file = "priors_psf.ini" + else: + priors_file = "priors.ini" - cl_root = sys.argv[6] - gaussian_part = sys.argv[7] # "iNKA" or "OneCovariance" - assert gaussian_part in ["iNKA", "OneCovariance"], "gaussian_part must be either 'iNKA' or 'OneCovariance'" + os.makedirs(args.output_config_dir, exist_ok=True) - pseudo_cl_file = output_folder+'/pseudo_cl_'+cl_root+'.fits' - pseudo_cl_cov_file = output_folder+f'/pseudo_cl_cov_g_ng_{gaussian_part}_{cl_root}.fits' + _generate_ini_file( + args, + template_base_realspace, + priors_file, + values_file, + suffix="", + is_harmonic=False, + ) - print("Creating 2PT fits extension...\n") - if not os.path.exists(pseudo_cl_file): - raise FileNotFoundError("Pseudo-Cl file not found. Please run cosmo_val.py first.") - pseudo_cl_hdu = pseudo_cl_to_fits(pseudo_cl_file) - print("Creating CovMat fits extension...\n") - cov_hdu = cov_pseudo_cl_to_fits(pseudo_cl_cov_file) + if args.cl_file: + template_base_harmonic = "cosmosis_pipeline_A_ia_cell.ini" - else: - arg_names += ["xi_root", "rhotau_stats", "tau_root", "covmat"] + values_file_cell = "values_ia.ini" + if args.mock: + priors_file_cell = "priors_mock_cell.ini" + else: + priors_file_cell = "priors.ini" + _generate_ini_file( + args, + template_base_harmonic, + priors_file_cell, + values_file_cell, + suffix="_cell", + is_harmonic=True, + ) + + +# ============================================================================ +# SECTION 4: MAIN WORKFLOW +# ============================================================================ + + +def parse_args(): + """Parse command-line arguments for unified data/mock interface.""" + parser = argparse.ArgumentParser( + description="Prepare CosmoSIS inference FITS files from real or mock data. " + "Supports multiple xi input formats (separate files, combined FITS).", + epilog=""" +Example for SP_v1.4.6_leak_corr (real data): + python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmosis_fitting.py \\ + --cosmosis-root "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --data-dir "/n09data/guerrini/output_chains/SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" \\ + --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ + --output-root "/home/guerrini/sp_validation/cosmo_inference" \\ + --xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/xi_plus_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/xi_minus_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_leak_corr_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked/covariance_SP_v1.4.6_leak_corr_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked_processed.txt" \\ + --use-rho-tau \\ + --rho-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/tau_stats_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --cov-tau "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_leak_corr_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" \\ + --cl-file "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_SP_v1.4.6_leak_corr.fits" \\ + --cov-cl "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_cov_g_ng_iNKA_SP_v1.4.6_leak_corr.fits" + +Example for glass mock v0 (mock data): + python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmosis_fitting.py \\ + --mock \\ + --cosmosis-root "glass_mock_v0_00001" \\ + --data-dir "/n09data/guerrini/glass_mock_chains/glass_mock_v0_00001" \\ + --nz-file "/n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt" \\ + --output-root "/home/guerrini/sp_validation/cosmo_inference" \\ + --output-basename "glass_mocks/v0/glass_mock_00001" \\ + --xi "/n09data/guerrini/glass_mock_v1.4.6/results/xi_glass_mock_00001_4096_nbins=20.fits" \\ + --cov-xi "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked/covariance_SP_v1.4.6_A_ng_minsep=1.0_maxsep=250.0_nbins=20_masked_processed.txt" \\ + --use-rho-tau \\ + --rho-stats "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" \\ + --tau-stats "/n17data/cdaley/unions/pure_eb/results/glass_mock_rhotau_samples/00001/tau_stats_sampled.fits" \\ + --cov-tau "/n17data/cdaley/unions/pure_eb/code/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/cov_tau_SP_v1.4.6_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_th.npy" \\ + --cl-file "/n09data/guerrini/glass_mock_v1.4.6/results/cl_glass_mock_00001_4096.npy" \\ + --cov-cl "/home/guerrini/sp_validation/notebooks/cosmo_val/output/pseudo_cl_cov_g_ng_iNKA_SP_v1.4.6.fits" + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--cosmosis-root", type=str, required=True, help="CosmoSIS root identifier" + ) + parser.add_argument( + "--data-dir", type=str, required=True, help="Output MCMC chain directory" + ) + parser.add_argument("--nz-file", type=str, required=True, help="Path to n(z) file") + parser.add_argument( + "--xi", + nargs="+", + required=True, + help="Xi files: 1 (mock FITS) or 2 (data: xi_plus.fits, xi_minus.fits)", + ) + parser.add_argument( + "--cov-xi", type=str, required=True, help="Xi covariance matrix file" + ) + parser.add_argument( + "--use-rho-tau", + action="store_true", + help="Include PSF systematics (requires --rho-stats, --tau-stats, --cov-tau)", + ) + parser.add_argument( + "--rho-stats", + type=str, + required=False, + help="Path to rho statistics FITS file (required if --use-rho-tau)", + ) + parser.add_argument( + "--tau-stats", + type=str, + required=False, + help="Path to tau statistics FITS file (required if --use-rho-tau)", + ) + parser.add_argument( + "--cov-tau", + type=str, + required=False, + help="Path to tau covariance matrix (required if --use-rho-tau)", + ) + parser.add_argument( + "--cl-file", + type=str, + required=False, + help="Path to C_ell data file (.npy, optional for data and mock)", + ) + parser.add_argument( + "--cov-cl", + type=str, + required=False, + help="Path to pseudo-C_ell covariance matrix (required if --cl-file)", + ) + parser.add_argument( + "--mock", action="store_true", help="Mock data mode" + ) + parser.add_argument( + "--output-root", + type=str, + required=True, + help=( + "Cosmo inference root directory (e.g., /home/guerrini/sp_validation/cosmo_inference). " + "FITS and config files are written under this root." + ), + ) + parser.add_argument( + "--output-basename", + type=str, + required=False, + help=( + "Optional override for the output sub-directory/basename inside --output-root. " + "Defaults to --cosmosis-root." + ), + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + try: + output_basename = args.output_basename or args.cosmosis_root + output_root_path = Path(args.output_root).expanduser().resolve() + output_basename_path = Path(output_basename) + data_dir_root = output_root_path / "data" / output_basename_path + config_dir_root = output_root_path / "cosmosis_config" + data_dir_root.mkdir(parents=True, exist_ok=True) + config_dir_root.mkdir(parents=True, exist_ok=True) + out_file_path = data_dir_root / f"cosmosis_{args.cosmosis_root}.fits" + args.output_root = str(output_root_path) + args.output_config_dir = str(config_dir_root) + args.output_basename = str(output_basename_path) + args.config_name_base = str(output_basename_path).replace("/", "_") + args.config_relative_fits = str( + Path("data") / output_basename_path / f"cosmosis_{args.cosmosis_root}.fits" + ) - for i, argv in enumerate(sys.argv[6:], start=6): - if i < len(arg_names): - print(f" argv[{i}] ({arg_names[i]}): {argv}") - else: - print(f" argv[{i}] (unexpected): {argv}") + print("=" * 60) + print("COSMOSIS_FITTING.PY") + print("=" * 60) + print(f"cosmosis_root: {args.cosmosis_root}") + print(f"data_dir: {args.data_dir}") + print(f"nz_file: {args.nz_file}") + print(f"output_root: {args.output_root}") + print(f"output_basename: {args.output_basename}") + print(f"out_file: {out_file_path}") + print(f"xi files: {args.xi}") + print(f"cov_xi: {args.cov_xi}") + print(f"use_rho_tau: {args.use_rho_tau}") + if args.use_rho_tau: + print(f"rho_stats: {args.rho_stats}") + print(f"tau_stats: {args.tau_stats}") + print(f"cov_tau: {args.cov_tau}") + if args.cl_file: + print(f"cl_file: {args.cl_file}") + if args.cov_cl: + print(f"cov_cl: {args.cov_cl}") print("=" * 60) print() - xi_root = sys.argv[6] - rhotau_stats = sys.argv[7] - tau_root = sys.argv[8] - cov_xi_file = sys.argv[9] #in cosmocov combined txt format + if args.use_rho_tau and not all([args.rho_stats, args.tau_stats, args.cov_tau]): + raise ValueError( + "--use-rho-tau requires: --rho-stats, --tau-stats, --cov-tau" + ) + if args.cl_file and not args.cov_cl: + raise ValueError("--cov-cl is required when --cl-file is provided") + if args.cov_cl and not args.cl_file: + raise ValueError("--cl-file is required when --cov-cl is provided") + + os.makedirs(args.data_dir, exist_ok=True) + print("Loading xi correlation functions...") + if args.mock: + # Mock mode expects a single combined FITS file + xip_hdu, xim_hdu = parse_combined_xi_fits(args.xi[0]) + else: + # Data mode expects two TreeCorr files (xi_plus, xi_minus) + if len(args.xi) != 2: + raise ValueError(f"Data mode requires exactly 2 xi files, got {len(args.xi)}") + xip_hdu, xim_hdu = treecorr_to_fits(args.xi[0], args.xi[1]) + + xi_theta = xip_hdu.data["ANG"] + print(f"Loaded xi: {len(xip_hdu.data)} bins") + + print("Loading covariance matrix...") + cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=None) + print(f"Loaded covariance: shape {cov_hdu.data.shape}") + + print("Loading n(z)...") + nz_hdu = nz_to_fits(args.nz_file) + print("Loaded n(z)") + + rho_hdu = None + tau_0_p_hdu = None + tau_2_p_hdu = None + if args.use_rho_tau: + print("Loading rho/tau statistics...") + tau_stats = fits.getdata(args.tau_stats) + rho_stats = fits.getdata(args.rho_stats) + tau_theta = tau_stats["theta"] + rho_theta = rho_stats["theta"] + + check_meanr_consistency(xi_theta, tau_theta, rho_theta, threshold=5.0) + print("✓ Forcing rho and tau to use xi meanr values for consistency") - two_pt_file_xip = output_folder+'/xi_plus_'+xi_root+'.fits' - two_pt_file_xim = output_folder+'/xi_minus_'+xi_root+'.fits' - #in cosmocov format - - use_tau_stats = sys.argv[7] - use_tau_stats = True if use_tau_stats == 'y' else False - rho_stats_file = output_folder+'/rho_tau_stats/rho_stats_'+tau_root+'.fits' if use_tau_stats else None - tau_stats_file = output_folder +'/rho_tau_stats/tau_stats_'+tau_root+'.fits' if use_tau_stats else None - cov_tau_file = output_folder + '/rho_tau_stats/cov_tau_'+tau_root+'_th.npy' if use_tau_stats else None + rho_hdu = rho_to_fits(args.rho_stats, theta=xi_theta) + tau_0_p_hdu, tau_2_p_hdu = tau_to_fits(args.tau_stats, theta=xi_theta) + print("Loaded rho/tau statistics") + + cov_hdu = covdat_to_fits(args.cov_xi, filename_cov_tau=args.cov_tau) + + cl_ee_hdu = None + cl_bb_hdu = None + if args.cl_file: + print("Loading Cl data...") + ell, cl_ee, cl_bb = load_pseudo_cl(args.cl_file) + print(f"Loaded Cl: {len(ell)} multipoles") + cl_ee_hdu, cl_bb_hdu = cl_to_fits(ell, cl_ee, cl_bb) + if args.use_rho_tau: + cov_hdu = covdat_to_fits( + args.cov_xi, + filename_cov_tau=args.cov_tau, + filename_cov_cl=args.cov_cl, + cov_cl_hdu="COVAR_FULL" + ) + else: + cov_hdu = covdat_to_fits( + args.cov_xi, + filename_cov_tau=None, + filename_cov_cl=args.cov_cl, + cov_cl_hdu="COVAR_FULL" + ) + print("Loaded pseudo-Cl covariance") + + pri_hdr = fits.Header() + pri_hdu = fits.PrimaryHDU(header=pri_hdr) + + print("Assembling FITS file...") + hdu_list = [pri_hdu, cov_hdu] + hdu_list.extend([nz_hdu, xip_hdu, xim_hdu]) + + if args.cl_file: + hdu_list.extend([cl_ee_hdu]) + + if args.use_rho_tau: + hdu_list.extend([tau_0_p_hdu, tau_2_p_hdu, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(str(out_file_path), overwrite=True) + print(f"✓ FITS file written to {out_file_path}") + print() - - #create the required FITS extensions - print("Creating 2PT fits extension...\n") - if not (os.path.exists(two_pt_file_xip) and os.path.exists(two_pt_file_xim)): - raise FileNotFoundError("2pt files not found. Please run cosmo_val.py first.") - xip_hdu, xim_hdu = treecorr_to_fits(two_pt_file_xip, two_pt_file_xim) - - # Extract xi meanr for consistency enforcement - xi_theta = xip_hdu.data['ANG'] # xi uses 'ANG' column for meanr - - if use_tau_stats and not os.path.exists(tau_stats_file): - raise Warning("Tau stats file not found. Please run cosmo_val.py first. Creating the FITS file without rho and tau statistics.") - use_tau_stats= False - cov_tau_file = None - if use_tau_stats: - - # Load original theta values for validation - tau_stats = fits.getdata(str(tau_stats_file)) - rho_stats = fits.getdata(str(rho_stats_file)) - tau_theta = tau_stats['theta'] - rho_theta = rho_stats['theta'] - - # Validate theta consistency and report differences - print("=" * 60) - print("MEANR CONSISTENCY CHECK") - print("=" * 60) - - # Calculate relative differences - tau_diff = np.abs((tau_theta - xi_theta) / xi_theta) * 100 - rho_diff = np.abs((rho_theta - xi_theta) / xi_theta) * 100 - - print(f"Xi theta range: {xi_theta.min():.6f} - {xi_theta.max():.6f} arcmin") - print(f"Tau theta range: {tau_theta.min():.6f} - {tau_theta.max():.6f} arcmin") - print(f"Rho theta range: {rho_theta.min():.6f} - {rho_theta.max():.6f} arcmin") - print() - print(f"Max tau-xi relative difference: {tau_diff.max():.3f}%") - print(f"Mean tau-xi relative difference: {tau_diff.mean():.3f}%") - print(f"Max rho-xi relative difference: {rho_diff.max():.3f}%") - print(f"Mean rho-xi relative difference: {rho_diff.mean():.3f}%") - - # Check for excessive differences - max_allowed_diff = 5.0 # 5% threshold - if tau_diff.max() > max_allowed_diff: - raise ValueError(f"Tau-xi meanr difference exceeds {max_allowed_diff}%: {tau_diff.max():.3f}%") - if rho_diff.max() > max_allowed_diff: - raise ValueError(f"Rho-xi meanr difference exceeds {max_allowed_diff}%: {rho_diff.max():.3f}%") - - print(f"✓ All differences below {max_allowed_diff}% threshold") - print("✓ Forcing rho and tau to use xi meanr values for consistency") - print("=" * 60) - print() - - print('Creating rho stats fits extension...\n') - rho_hdu = rho_to_fits(rho_stats_file) - print("Creating tau fits extensions...\n") - tau_0_p_hdu, tau_2_p_hdu = tau_to_fits(tau_stats_file) - print("Creating CovMat fits extension...\n") - cov_hdu = covdat_to_fits(cov_xi_file, cov_tau_file) - print("Creating n(z) fits extension...\n") - nz_hdu = nz_to_fits(nz_file) - - # create header for primary HDU - pri_hdr_dict = {} - - pri_hdr = fits.Header() - for key in pri_hdr_dict: - pri_hdr[key] = pri_hdr_dict[key] - - # create primary HDU - pri_hdu = fits.PrimaryHDU(header=pri_hdr) - - # create final FITS HDU - print("Writing out combined FITS file...\n") - if use_pseudo_cell: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, pseudo_cl_hdu] - else: - if use_tau_stats: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu, tau_0_p_hdu, tau_2_p_hdu, rho_hdu] - else: - hdu_list = [pri_hdu, cov_hdu, nz_hdu, xip_hdu, xim_hdu] - hdul = fits.HDUList(hdu_list) - hdul.writeto(out_file, overwrite=True) - print("FITS file written out to %s" % out_file) + generate_cosmosis_config(args) + + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 67a0cad..bb791ef 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -1,2267 +1,904 @@ -paths: - output: ./output - -nz: - subdir: CFIS/v1.0/nz - dndz: - path: dndz - blind: A - -SP_v1.0: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: solid - colour: blue - marker: o - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_2022_v1.0.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - -SP_v1.1: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: dashed - colour: b - marker: s - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_2022_v1.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - +DES: + subdir: /n17data/mkilbing/astro/data/DES + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: '*' + cov_th: + A: 4143 + n_e: 5.67 + n_psf: 3.8 + sigma_e: 0.29 + path_rho: rho_stats_DES.fits + path_tau: tau_stats_DES.fits + path_xi_plus: xi_plus_DES.fits + path_xi_minus: xi_minus_DES.fits + psf: + PSF_size: piff_T + square_size: false + star_size: obs_T + hdu: 1 + path: psf_y3a1-v29.fits + ra_col: ra + dec_col: dec + e1_PSF_col: piff_e1 + e1_star_col: obs_e1 + e2_PSF_col: piff_e2 + e2_star_col: obs_e2 + shear: + R: 1.0 + R11: R11 + R22: R22 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: DES_Y3_cut.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_psf + e2_col: e2 + e2_PSF_col: e2_psf + label: DES-Y3 + star: + ra_col: ra + dec_col: dec + e1_col: obs_e1 + e2_col: obs_e2 + path: psf_y3a1-v29.fits +SP_axel_v0.0: + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dotted + marker: '' + cov_th: + A: 1500 + n_e: 6.47 + n_psf: 0.47 + sigma_e: 0.3 + path_rho: rho_stats_SP_v0.0.fits + path_tau: tau_stats_SP_v0.0.fits + path_xi_plus: xi_plus_SP_v0.0.fits + path_xi_minus: xi_minus_SP_v0.0.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: star_cat.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: shapepipe_1500_goldshape_v1.fits + w_col: w + e1_col: g1 + e1_PSF_col: E1_PSF_HSM + e2_col: g2 + e2_PSF_col: E2_PSF_HSM + star: + ra_col: RA + dec_col: DEC + e1_col: E1_STAR_HSM + e2_col: E2_STAR_HSM + path: star_cat.fits +SP_test: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.1.fits +SP_v0.1.1: + subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: v + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: star_cat.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: DEC + e1_col: E1_STAR_HSM + e2_col: E2_STAR_HSM + path: star_cat.fits SP_v1.3: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: solid - colour: green - marker: p - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_extended_2022_v1.3.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4_conv: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: solid - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4_noalpha: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3135.3602943406763 #deg^2 - n_e: 7.853461190377332 #arcmin-2 - n_psf: 0.5807322129800436 #arcmin-2 - sigma_e: 0.31092113426960205 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - - + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: p + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: T_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: T_STAR_HSM + hdu: 1 + path: unions_shapepipe_star_2022_v1.3.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_v1.3.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.0.3.fits SP_v1.4-P3: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P3 - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 337 #deg^2 - n_e: 7.45 #arcmin-2 - n_psf: 0.34 #arcmin-2 - sigma_e: 0.32 - path_rho: rho_stats_SP_v1.4-P3.fits - path_tau: tau_stats_SP_v1.4-P3.fits - path_xi_plus: xi_plus_SP_v1.4-P3.fits - path_xi_minus: xi_minus_SP_v1.4-P3.fits - shear: - path: unions_shapepipe_extended_2022_P3_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P1+3.fits - path_tau: tau_stats_SP_v1.4-P1+3.fits - path_xi_plus: xi_plus_SP_v1.4-P1+3.fits - path_xi_minus: xi_minus_SP_v1.4-P1+3.fits - shear: - path: unions_shapepipe_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P1+3_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_wcs: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_no_alpha: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_rmalpha_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3_li_2024: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3 - ls: dashed - colour: gold - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1133 #deg^2 - n_e: 7.67 #arcmin-2 - n_psf: 0.41 #arcmin-2 - sigma_e: 0.31 - shear: - path: unions_shapepipe_extended_rmalpha_li_2022_P1+3_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3_v1.4.fits - hdu: 2 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 - ls: dashed - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4_wcs: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_extended_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4-P1+3+4_no_alpha: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/P1+3+4 - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_extended_rmalpha_2022_P1+3+4_v1.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_P1+3+4_v1.4.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_P1+3+4_v1.4.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.2: - pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1826 #deg^2 - n_e: 7.75 #arcmin-2 - n_psf: 0.35 #arcmin-2 - sigma_e: 0.3133 - shear: - path: unions_shapepipe_cut_struc_2024_v1.4.2.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe/P3 + pipeline: SP + colour: brown + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 337 + n_e: 7.45 + n_psf: 0.34 + sigma_e: 0.32 + path_rho: rho_stats_SP_v1.4-P3.fits + path_tau: tau_stats_SP_v1.4-P3.fits + path_xi_plus: xi_plus_SP_v1.4-P3.fits + path_xi_minus: xi_minus_SP_v1.4-P3.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 2 + path: unions_shapepipe_psf_2022_P3_v1.4.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_P3_v1.4.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_P3_v1.4.fits SP_v1.4.1_noleakage: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 3191.1827474082734 #deg^2 - n_e: 7.0582373812033286 #arcmin-2 - n_psf: 0.5705736293858423 #arcmin-2 - sigma_e: 0.317469332009887 - shear: - path: unions_shapepipe_2024_v1.4.1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 3191.1827474082734 + n_e: 7.0582373812033286 + n_psf: 0.5705736293858423 + sigma_e: 0.317469332009887 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.1.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_2024_v1.4.1.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.1.fits +SP_v1.4.2: + subdir: /n17data/mkilbing/astro/data/UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.2 + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 1826 + n_e: 7.75 + n_psf: 0.35 + sigma_e: 0.3133 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.2.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: dodgerblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: ../../../../../../sguerrini/unions_shapepipe_cut_struc_2024_v1.4.5_rerun.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - mask: /n09data/guerrini/glass_mock/mask_v1.4.5_nside8192.fits - redshift_distr: ../../../sguerrini/UNIONS/WL/nz/v1.4.5/nz_SP_v1.4.5_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.6: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - mask: /n09data/guerrini/glass_mock/mask_v1.4.5_nside8192.fits - redshift_distr: ../../../sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.8: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: red - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - mask: /n09data/guerrini/glass_mock/mask_v1.4.5_nside8192.fits - redshift_distr: ../../../sguerrini/UNIONS/WL/nz/v1.4.8/nz_SP_v1.4.8_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5.A: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: red - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: g1 - e2_col: g2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_bright: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: mediumblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_faint: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: forestgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_intermediate: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: orange - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.6_leak_corr: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: olive - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - mask: /n09data/guerrini/glass_mock/mask_v1.4.5_nside8192.fits - redshift_distr: ../../../sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.7_leak_corr: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: darkorchid - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.8_leak_corr: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: peachpuff - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../UNIONS/WL/v1.4.x/v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - mask: /n09data/guerrini/glass_mock/mask_v1.4.5_nside8192.fits - redshift_distr: ../../../sguerrini/UNIONS/WL/nz/v1.4.8/nz_SP_v1.4.8_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: dodgerblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2582.310451583689 + n_e: 7.748708180391998 + n_psf: 0.4973044366748657 + sigma_e: 0.4375145387558129 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.5_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5.A: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: red - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5/shapepipe_SPv1.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: g1 - e2_col: g2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - + subdir: /n17data/guinot/CFIS_3500_cat/catalogues_SPv1_v1.4.5 + pipeline: SP + colour: red + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: shapepipe_SPv1.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: g1 + e1_PSF_col: e1_PSF + e2_col: g2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_bright: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: mediumblue - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: mediumblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_faint: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: forestgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_intermediate: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: orange - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: ../../../../../../murray/unions_cats/unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w: w_des - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: v1.4.x/v1.4.5/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: v1.4.x/v1.4.5/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - - + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: forestgreen + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4.5_glass_mock: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe - ls: dashed - colour: crimson - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: /v1.4.5_glass_mock/unions_glass_sim_00001_4096.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_test: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - ls: dashed - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2024_v1.4.1.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.4.1.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.4.5_leak_corr: - pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ - marker: d - ls: dashed - colour: blue - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2782 #deg^2 - n_e: 7.18 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.4370966656902571 - shear: - path: ../../../../../../sguerrini/unions_shapepipe_cut_struc_2024_v1.4.5_rerun.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_des - R: 1.0 - mask: /n09data/guerrini/glass_mock/mask_v1.4.5_nside8192.fits - redshift_distr: ../../../sguerrini/UNIONS/WL/nz/v1.4.5/nz_SP_v1.4.5_A.txt - star: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: ../../../../../../UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.5.4: - pipeline: SP - subdir: ../../../UNIONS/WL/v1.5.x - ls: dashed - colour: green - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.5.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.5.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.5.4_leak_corr: - pipeline: SP - subdir: ../../../UNIONS/WL/v1.5.x - ls: dashed - colour: darkgreen - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2420.2651014497287 #deg^2 - n_e: 7.040818382014773 #arcmin-2 - n_psf: 0.752316232272063 #arcmin-2 - sigma_e: 0.30961528707207325 - shear: - path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_leak_corrected - e2_col: e2_leak_corrected - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w_iv - R: 1.0 - star: - path: unions_shapepipe_star_2024_v1.5.a.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2024_v1.5.a.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -LF_v1.0: - pipeline: LF - subdir: CFIS/v1.0/Lensfit - ls: solid - colour: r - marker: "8" - getdist_colour: 1.0, 0.5, 0.0 - shear: - path: lensfit_goldshape_2022v1.fits - covmat_file: ./covs/lensfit_A/cov_lensfit_A.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: full_starcat_tmp_1_10_seed_1234.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - -LF_v2.0: - pipeline: LF - subdir: CFIS/v2.0/Lensfit - ls: dashed - colour: r - marker: p - shear: - path: lensfit_goldshape_2022v2_psf1.fits - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: Pall_hdu1_THELI_psfs1_mosaic.fits - ra_col: RA - dec_col: Dec - e1_col: e1_psf - e2_col: e2_psf - -SP_matched_LF_v1.0: - pipeline: SP - subdir: CFIS/v1.0/matched_LF_SP - ls: dotted - colour: magenta - marker: v - getdist_colour: 0.90340530805574, 0.06198853997875753, 0.6527408678603205 - shear: - path: masked_matched_unions_shapepipe_extended_2022_v1.0.fits - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -LF_matched_SP_v1.0: - pipeline: LF - subdir: CFIS/v1.0/matched_LF_SP - ls: dashdot - colour: magenta - marker: ^ - getdist_colour: 0.0, 0.5, 0.0 - shear: - path: masked_lensfit_goldshape_2022v1.fits - covmat_file: ./covs/shapepipe_matched/cov_shapepipe_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: full_starcat_tmp_1_10_seed_1234.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - -SP_v1.0_LFmask_4k: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dashdot - colour: blue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_2022_v1.0_mtheli4k.fits - e1_col: e1 - e2_col: e2 - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.0_LFmask_8k: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: blue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_extended_2022_v1.0_mtheli8k.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli8k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.3_LFmask_4k: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dashdot - colour: green - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: unions_shapepipe_2022_v1.3_mtheli4k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.0.2_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -SP_v1.4-P3_LFmask: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask/P3 - ls: dashdot - colour: black - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 223 #deg^2 - n_e: 7.44 #arcmin-2 - n_psf: 0.32 #arcmin-2 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.4-P3_LFmask.fits - path_tau: tau_stats_SP_v1.4-P3_LFmask.fits - path_xi_plus: xi_plus_SP_v1.4-P3_LFmask.fits - path_xi_minus: xi_minus_SP_v1.4-P3_LFmask.fits - shear: - path: unions_shapepipe_2022_v1.4_mtheli4k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4_mtheli4k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_2022_v1.4_mtheli4k.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_v1.3_LFmask_8k: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: green - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_no_alpha: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: blueviolet - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.3_mtheli8k.fits - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_li_2024: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: deepskyblue - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 7.6 - n_psf: 0.68 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_rmalpha_li_2022_v1.3_mtheli8k.fits - e1_col: e1_cor_2 - e2_col: e2_cor_2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_SN8: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: darkgreen - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 9.7 - n_psf: 0.48 - sigma_e: 0.35 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_sn8_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_SN7: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: purple - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 10.989 - n_psf: 0.48 - sigma_e: 0.3796 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_sn7_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - -SP_v1.3_LFmask_8k_F2: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: dotted - colour: lawngreen - marker: h - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - cov_th: - A: 2138 - n_e: 9.4 - n_psf: 0.48 - sigma_e: 0.31 - path_rho: rho_stats_SP_v1.3_LFmask_8k.fits - path_tau: tau_stats_SP_v1.3_LFmask_8k.fits - path_xi_plus: xi_plus_SP_v1.3_LFmask_8k.fits - path_xi_minus: xi_minus_SP_v1.3_LFmask_8k.fits - shear: - path: unions_shapepipe_extended_2022_f2_v1.3_mtheli8k.fits - e1_col: e1 - e2_col: e2 - w_col: w - R: 1.0 - R11: 1.0 - R22: 1.0 - star: - path: unions_shapepipe_star_2022_v1.0.3_mtheli8k.fits - label: SP_LFmask_star - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_star_2022_v1.3_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: T_PSF_HSM - star_size: T_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: False - + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: crimson + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2782 + n_e: 7.18 + n_psf: 0.752316232272063 + sigma_e: 0.4370966656902571 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n09data/guerrini/glass_mock/results/unions_glass_sim_00000_4096.fits + path_template: /n09data/guerrini/glass_mock/results/unions_glass_sim_{seed:05d}_4096.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.6_glass_mock: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkgreen + getdist_colour: 0.0, 0.7, 0.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00001_4096.fits + path_template: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_{seed:05d}_4096.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.5_intermediate: + subdir: /n17data/murray/unions_cats + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.6: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.7: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkorchid + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2050.5970269635545 + n_e: 6.121543643587931 + n_psf: 0.752316232272063 + sigma_e: 0.3797419081703805 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.7_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.8: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkorchid + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2049.034211379564 + n_e: 6.121484916053109 + n_psf: 0.752316232272063 + sigma_e: 0.37973978258052915 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.8_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits SP_v1.4_LFmask_8k: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: solid - colour: black - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2137.7977618140676 #deg^2 - n_e: 7.976462506484096 #arcmin-2 - n_psf: 0.5434016250405327 #arcmin-2 - sigma_e: 0.31509572849714534 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: black + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: d + cov_th: + A: 2137.7977618140676 + n_e: 7.976462506484096 + n_psf: 0.5434016250405327 + sigma_e: 0.31509572849714534 + path_rho: rho_stats_SP_v1.4.fits + path_tau: tau_stats_SP_v1.4.fits + path_xi_plus: xi_plus_SP_v1.4.fits + path_xi_minus: xi_minus_SP_v1.4.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_2022_v1.4.0_mtheli8k.fits + w_col: w + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits SP_v1.4_LFmask_8k_noalpha: - pipeline: SP - subdir: CFIS/v1.0/SP_LFmask - ls: solid - colour: brown - marker: d - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 2137.7977618140676 #deg^2 - n_e: 7.976462506484096 #arcmin-2 - n_psf: 0.5434016250405327 #arcmin-2 - sigma_e: 0.31509572849714534 - path_rho: rho_stats_SP_v1.4.fits - path_tau: tau_stats_SP_v1.4.fits - path_xi_plus: xi_plus_SP_v1.4.fits - path_xi_minus: xi_minus_SP_v1.4.fits - shear: - path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - e1_col: e1_cor - e2_col: e2_cor - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - w_col: w - R: 1.0 - star: - path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits - ra_col: RA - dec_col: Dec - e1_col: e1 - e2_col: e2 - psf: - path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits - label: SP_LFmask_psf - hdu : 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: "FLAG_PSF_HSM" - star_flag: "FLAG_STAR_HSM" - square_size: True - -SP_matched_MP_v1.0: - pipeline: SP - subdir: CFIS/v1.0/matched_SP_MP - ls: dashdot - colour: lightblue - marker: D - getdist_colour: 0.04606552309790213, 0.9780505945532256, 0.37684692395248665 - shear: - path: cfis-shapepipe.parquet - e1_col: e1 - e2_col: e2 - R: 1.0 - -SP_matched_LF_fp_v1.0: - pipeline: SP - ls: dashed - colour: lightblue - marker: < - getdist_colour: 0.08130806428547999, 0.1463709449188496, 0.674947827197815 - shear: - path: matched_footprint_shapepipe.fits - covmat_file: ./covs/lensfit_matched/cov_lensfit_matched.txt - e1_col: e1 - e2_col: e2 - R: 1.0 - -SP_axel_v0.0: - pipeline: SP - subdir: CFIS/v0.0 - ls: dotted - colour: cyan - marker: > - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 1500 - n_e: 6.47 - n_psf: 0.47 - sigma_e: 0.30 - path_rho: rho_stats_SP_v0.0.fits - path_tau: tau_stats_SP_v0.0.fits - path_xi_plus: xi_plus_SP_v0.0.fits - path_xi_minus: xi_minus_SP_v0.0.fits - shear: - path: shapepipe_1500_goldshape_v1.fits - e1_col: g1 - e2_col: g2 - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - w_col: w - R: 1.0 - redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - star: - path: star_cat.fits - ra_col: RA - dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM - psf: - path: star_cat.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - square_size: True - -SP_v0.1.1: - pipeline: SP - subdir: CFIS/v0.0 - ls: dashdot - colour: cyan - marker: v - getdist_colour: 0.0, 0.5, 1.0 - shear: - path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_PSF - e2_PSF_col: e2_PSF - R: 1.0 - star: - path: star_cat.fits - ra_col: RA - dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM - psf: - path: star_cat.fits - hdu: 1 - ra_col: RA - dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e2_PSF_col: E2_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_star_col: E2_STAR_HSM - PSF_size: SIGMA_PSF_HSM - star_size: SIGMA_STAR_HSM - PSF_flag: FLAG_PSF_HSM - star_flag: FLAG_STAR_HSM - -DES: - pipeline: SP - subdir: DES - ls: solid - colour: orange - marker: "*" - getdist_colour: 0.0, 0.5, 1.0 - cov_th: - A: 4143 - n_e: 5.67 - n_psf: 3.8 - sigma_e: 0.29 - path_rho: rho_stats_DES.fits - path_tau: tau_stats_DES.fits - path_xi_plus: xi_plus_DES.fits - path_xi_minus: xi_minus_DES.fits - shear: - path: DES_Y3_cut.fits - label: DES-Y3 - e1_col: e1 - e2_col: e2 - e1_PSF_col: e1_psf - e2_PSF_col: e2_psf - w_col: w - R: 1.0 - R11: R11 - R22: R22 - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - star: - path: psf_y3a1-v29.fits - ra_col: ra - dec_col: dec - e1_col: obs_e1 - e2_col: obs_e2 - psf: - path: psf_y3a1-v29.fits - hdu: 1 - ra_col: ra - dec_col: dec - e1_PSF_col: piff_e1 - e2_PSF_col: piff_e2 - e1_star_col: obs_e1 - e2_star_col: obs_e2 - PSF_size: piff_T - star_size: obs_T - square_size: False + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask + pipeline: SP + colour: brown + getdist_colour: 0.0, 0.5, 1.0 + ls: solid + marker: d + cov_th: + A: 2137.7977618140676 + n_e: 7.976462506484096 + n_psf: 0.5434016250405327 + sigma_e: 0.31509572849714534 + path_rho: rho_stats_SP_v1.4.fits + path_tau: tau_stats_SP_v1.4.fits + path_xi_plus: xi_plus_SP_v1.4.fits + path_xi_minus: xi_minus_SP_v1.4.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + label: SP_LFmask_psf + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: unions_shapepipe_extended_rmalpha_2022_v1.4.0_mtheli8k.fits + w_col: w + e1_col: e1_cor + e1_PSF_col: e1_PSF + e2_col: e2_cor + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2022_v1.4.0_mtheli8k.fits +SP_v1.5.4: + subdir: /n17data/UNIONS/WL/v1.5.x + pipeline: SP + colour: green + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2420.2651014497287 + n_e: 7.040818382014773 + n_psf: 0.752316232272063 + sigma_e: 0.30961528707207325 + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.5.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits + w_col: w_iv + e1_col: e1 + e1_PSF_col: e1_PSF + e2_col: e2 + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.5.a.fits +nz: + subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/nz + dndz: + blind: A + path: dndz +paths: + output: ./output diff --git a/notebooks/cosmo_val/compute_theory_cov.py b/notebooks/cosmo_val/compute_theory_cov.py index 2c93d47..8c20a73 100644 --- a/notebooks/cosmo_val/compute_theory_cov.py +++ b/notebooks/cosmo_val/compute_theory_cov.py @@ -41,7 +41,7 @@ def get_params_rho_tau(cat, survey="other"): base_dir = '/home/guerrini/data/' - versions = ['SP_v1.4-P3', 'SP_v1.4-P3_LFmask', 'SP_v1.4-P1+3', 'SP_v1.3_LFmask_8k', 'SP_axel_v0.0', 'DES'] + versions = ['SP_v1.4-P3', 'SP_v1.4-P3_LFmask', 'SP_v1.3_LFmask_8k', 'SP_axel_v0.0', 'DES'] path_config = '/home/guerrini/sp_validation/notebooks/cosmo_val/cat_config.yaml' output_dir = '/home/guerrini/sp_validation/notebooks/cosmo_val/output/rho_tau_stats/' @@ -94,4 +94,4 @@ def get_params_rho_tau(cat, survey="other"): print("--- Covariance computation %s seconds ---" % (time.time() - start_time)) np.save(output_dir+'/cov_tau_'+ver+'_th.npy', cov) - print("Saved covariance matrix of version: ", ver) \ No newline at end of file + print("Saved covariance matrix of version: ", ver) diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index e933904..f08f7c0 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -19,13 +19,14 @@ # %% cv = CosmologyValidation( - versions=["SP_v1.4.5", "SP_v1.4.6", "SP_v1.4.8", "SP_v1.4.5_leak_corr", "SP_v1.4.6_leak_corr", "SP_v1.4.8_leak_corr"], - data_base_dir="/n17data/mkilbing/astro/data/", + versions=["SP_v1.4.5_leak_corr", "SP_v1.4.6_leak_corr", "SP_v1.4.7_leak_corr"], + npatch=1, theta_min=1.0, theta_max=250.0, + nbins=20, theta_min_plot=0.8, theta_max_plot=260.0, - npatch=100, + ylim_alpha=[-0.01, 0.05], nrandom_cell=100, cell_method="catalog", nside_mask=8192, diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index d24f9e1..09ce030 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -659,6 +659,7 @@ def plot_pure_eb_correlations( ax.set(xscale="log", xlabel=r"$\theta$ [arcmin]") ax.axhline(0, alpha=0.3, color="k", linestyle="--", linewidth=0.5) ax.legend(loc="upper left") + ax.set_ylim(-0.5, 2) # Save the axis limits after data plotting but before adding gray regions original_xlims = [ax.get_xlim() for ax in axs] @@ -1037,6 +1038,7 @@ def plot_cosebis_modes( plt.legend() plt.xlabel("n (mode)") plt.ylabel("E_n, B_n") + plt.ylim(-0.5e-10, 3e-10) # Add scale cut information to title - use actual scale cut from results scale_info = "" @@ -1092,4 +1094,3 @@ def plot_cosebis_covariance_matrix(results, version, var_method, output_path): ax.set_title(f"{version} COSEBIs E/B {var_method} correlation matrix") plt.savefig(output_path, dpi=300, bbox_inches="tight") - diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 3b8730c..d3a46ea 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,5 +1,7 @@ # %% +import copy import os +from pathlib import Path import configparser import colorama @@ -19,15 +21,169 @@ from .cosmology import get_cosmo, get_theo_c_ell from .plots import FootprintPlotter -from .rho_tau import get_params_rho_tau, get_rho_tau_w_cov, get_samples +from .rho_tau import ( + get_params_rho_tau, + get_rho_tau_w_cov, + get_samples, +) # %% class CosmologyValidation: + """Framework for cosmic shear validation and systematics analysis. + + Handles two-point correlation function measurements, PSF systematics (rho/tau), + pseudo-C_ell analysis, and covariance estimation for weak lensing surveys. + Supports multiple catalog versions with automatic leakage-corrected variants. + + Parameters + ---------- + versions : list of str + Catalog version identifiers to analyze. Appending '_leak_corr' to a base + version creates a virtual catalog using leakage-corrected ellipticity columns + (e1_col_corrected/e2_col_corrected) from the base version configuration. + catalog_config : str, default './cat_config.yaml' + Path to catalog configuration YAML defining survey metadata, file paths, + and analysis settings for each version. + output_dir : str, optional + Override for output directory. If None, uses catalog config's paths.output. + rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' + Fitting method for PSF leakage systematics parameters. + cov_estimate_method : {'th', 'jk'}, default 'th' + Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife. + compute_cov_rho : bool, default True + Whether to compute covariance for rho statistics during PSF analysis. + n_cov : int, default 100 + Number of realizations for covariance estimation when using theory method. + theta_min : float, default 0.1 + Minimum angular separation in arcminutes for correlation function binning. + theta_max : float, default 250 + Maximum angular separation in arcminutes for correlation function binning. + nbins : int, default 20 + Number of angular bins for TreeCorr real-space correlation functions. + var_method : {'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'}, default 'jackknife' + TreeCorr variance estimation method. + npatch : int, default 20 + Number of spatial patches for jackknife variance estimation. + quantile : float, default 0.1587 + Quantile for uncertainty bands in plots (default: 1-sigma ≈ 0.159). + theta_min_plot : float, default 0.08 + Minimum angular scale for plotting (may differ from analysis cut). + theta_max_plot : float, default 250 + Maximum angular scale for plotting. + ylim_alpha : list of float, default [-0.005, 0.05] + Y-axis limits for alpha systematic parameter plots. + ylim_xi_sys_ratio : list of float, default [-0.02, 0.5] + Y-axis limits for xi systematics ratio plots. + nside : int, default 1024 + HEALPix resolution for pseudo-C_ell analysis and area computation. + binning : {'powspace', 'linspace', 'logspace'}, default 'powspace' + Ell binning scheme for pseudo-C_ell (powspace = ell^power spacing). + power : float, default 0.5 + Exponent for power-law binning when binning='powspace'. + n_ell_bins : int, default 32 + Number of ell bins for pseudo-C_ell analysis. + pol_factor : bool, default True + Apply polarization correction factor in pseudo-C_ell calculations. + nrandom_cell : int, default 10 + Number of random realizations for C_ell error estimation. + cosmo_params : dict, optional + Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + + Attributes + ---------- + versions : list of str + Validated catalog versions after processing _leak_corr variants. + cc : dict + Loaded catalog configuration with resolved absolute paths. + catalog_config_path : Path + Resolved path to the catalog configuration file. + treecorr_config : dict + Configuration dictionary passed to TreeCorr correlation objects. + cosmo : pyccl.Cosmology + Cosmology object for theory predictions. + + Notes + ----- + - Path resolution: Relative paths in catalog config are resolved using each + version's 'subdir' field as the base directory. + - Virtual _leak_corr versions: These create deep copies of the base version + config, swapping e1_col/e2_col with e1_col_corrected/e2_col_corrected. + - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, + 'simple' otherwise, following TreeCorr best practices. + """ + + @staticmethod + def _split_seed_variant(version): + """Return the base version and seed label if version encodes a seed.""" + if "_seed" not in version: + return None, None + base, seed_label = version.rsplit("_seed", 1) + if not base or not seed_label.isdigit(): + return None, None + return base, seed_label + + @staticmethod + def _materialize_seed_path( + base_cfg, seed_label, version, base_version, catalog_config + ): + """Render the seed-specific shear path using Python string formatting.""" + shear_cfg = base_cfg["shear"] + template = shear_cfg.get("path_template") + + try: + seed_value = int(seed_label) + except ValueError as error: + raise ValueError( + f"Seed suffix for '{version}' is not numeric; cannot materialize path." + ) from error + + format_context = {"seed": seed_value, "seed_label": seed_label} + + if template: + try: + return template.format(**format_context) + except KeyError as error: + raise KeyError( + f"Missing placeholder '{error.args[0]}' in path_template for " + f"'{base_version}' while materializing '{version}'. Update " + f"{catalog_config}." + ) from error + except ValueError as error: + raise ValueError( + f"Invalid format specification in path_template for '{base_version}' " + f"while materializing '{version}'." + ) from error + + path = shear_cfg.get("path", "") + token_start = path.rfind("seed") + if token_start == -1: + raise ValueError( + f"Cannot materialize '{version}': '{base_version}' lacks a shear " + f"path_template and its shear path '{path}' does not contain a 'seed' " + f"token. Update {catalog_config}." + ) + cursor = token_start + 4 # len("seed") + if cursor < len(path) and not path[cursor].isdigit(): + cursor += 1 + digit_start = cursor + while cursor < len(path) and path[cursor].isdigit(): + cursor += 1 + digit_end = cursor + digits = path[digit_start:digit_end] + if not digits: + raise ValueError( + f"Cannot materialize '{version}': shear path '{path}' for base version " + f"'{base_version}' lacks digits after the seed token. Update " + f"{catalog_config}." + ) + + template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" + return template.format(**format_context) + def __init__( self, versions, - data_base_dir, catalog_config="./cat_config.yaml", output_dir=None, rho_tau_method="lsq", @@ -54,10 +210,7 @@ def __init__( nrandom_cell=10, path_onecovariance=None, cosmo_params=None, - redshift_file=None ): - self.versions = versions - self.data_base_dir = data_base_dir self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method self.compute_cov_rho = compute_cov_rho @@ -92,8 +245,6 @@ def __init__( # Use Planck 2018 defaults self.cosmo = get_cosmo() - # load redshift distribution from file if provided - self.z_dist = np.loadtxt(redshift_file) if redshift_file is not None else None self.treecorr_config = { "ra_units": "degrees", @@ -106,18 +257,78 @@ def __init__( "cross_patch_weight": "match" if var_method == "jackknife" else "simple", } - with open(catalog_config, "r") as file: + self.catalog_config_path = Path(catalog_config) + with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) - for ver in ["nz", *versions]: - if ver not in cc: - raise KeyError( - f"Version string {ver} not found in config file{catalog_config}" - ) - version_base = f"{data_base_dir}/{cc[ver]['subdir']}" + def resolve_paths_for_version(ver): + """Resolve relative paths for a version using its subdir.""" + subdir = Path(cc[ver]["subdir"]) for key in cc[ver]: if "path" in cc[ver][key]: - cc[ver][key]["path"] = f"{version_base}/{cc[ver][key]['path']}" + path = Path(cc[ver][key]["path"]) + cc[ver][key]["path"] = ( + str(path) if path.is_absolute() else str(subdir / path) + ) + + resolve_paths_for_version("nz") + processed = {"nz"} + final_versions = [] + leak_suffix = "_leak_corr" + + def ensure_version_exists(ver): + if ver in processed: + return + + if ver in cc: + resolve_paths_for_version(ver) + processed.add(ver) + return + + seed_base, seed_label = self._split_seed_variant(ver) + + if ver.endswith(leak_suffix): + base_ver = ver[: -len(leak_suffix)] + ensure_version_exists(base_ver) + shear_cfg = cc[base_ver]["shear"] + if "e1_col_corrected" not in shear_cfg or "e2_col_corrected" not in shear_cfg: + raise ValueError( + f"{base_ver} does not have e1_col_corrected/e2_col_corrected " + f"fields; cannot create {ver}" + ) + if ver not in cc: + cc[ver] = copy.deepcopy(cc[base_ver]) + cc[ver]["shear"]["e1_col"] = shear_cfg["e1_col_corrected"] + cc[ver]["shear"]["e2_col"] = shear_cfg["e2_col_corrected"] + resolve_paths_for_version(ver) + processed.add(ver) + return + + if seed_base is not None: + ensure_version_exists(seed_base) + if ver not in cc: + cc[ver] = copy.deepcopy(cc[seed_base]) + seed_path = self._materialize_seed_path( + cc[seed_base], + seed_label, + ver, + seed_base, + catalog_config, + ) + cc[ver]["shear"]["path"] = seed_path + resolve_paths_for_version(ver) + processed.add(ver) + return + + raise KeyError( + f"Version string {ver} not found in config file {catalog_config}" + ) + + for ver in versions: + ensure_version_exists(ver) + final_versions.append(ver) + + self.versions = final_versions # Override output directory if provided if output_dir is not None: @@ -126,6 +337,153 @@ def __init__( if not os.path.exists(cc["paths"]["output"]): os.mkdir(cc["paths"]["output"]) + def get_redshift(self, version): + """Load redshift distribution for a catalog version. + + Parameters + ---------- + version : str + Catalog version identifier + + Returns + ------- + z : ndarray + Redshift values + nz : ndarray + n(z) probability density + """ + redshift_path = self.cc[version]["shear"]["redshift_path"] + return np.loadtxt(redshift_path, unpack=True) + + def compute_survey_stats( + self, + ver, + weights_key_override=None, + mask_path=None, + nside=None, + overwrite_config=False, + ): + """Compute effective survey statistics for a catalog version. + + Parameters + ---------- + ver : str + Version string registered in the catalog config. + weights_key_override : str, optional + Override the weight column key (defaults to the configured `w_col`). + mask_path : str, optional + Explicit mask path to use when measuring survey area. + nside : int, optional + If provided, compute survey area from the catalog using this NSIDE when no + mask path is available. + overwrite_config : bool, optional + If True, persist the derived statistics back to the catalog configuration. + + Returns + ------- + dict + Dictionary containing: + - area_deg2: Survey area in square degrees. + - n_eff: Effective number density per arcmin^2. + - sigma_e: Per-component shape noise. + - sum_w: Sum of weights. + - sum_w2: Sum of squared weights. + - catalog_size: Number of galaxies processed. + """ + if ver not in self.cc: + raise KeyError(f"Version {ver} not found in catalog configuration") + + shear_cfg = self.cc[ver]["shear"] + cov_th = self.cc[ver].get("cov_th", {}) + + if "path" not in shear_cfg: + raise KeyError(f"No shear catalog path defined for version {ver}") + + catalog_path = shear_cfg["path"] + if not os.path.exists(catalog_path): + raise FileNotFoundError(f"Shear catalog not found: {catalog_path}") + + data = fits.getdata(catalog_path, memmap=True) + n_rows = len(data) + + e1 = np.asarray(data[shear_cfg["e1_col"]], dtype=float) + e2 = np.asarray(data[shear_cfg["e2_col"]], dtype=float) + + weight_column = weights_key_override or shear_cfg["w_col"] + if weight_column not in data.columns.names: + raise KeyError(f"Weight column '{weight_column}' missing in {catalog_path}") + + w = np.asarray(data[weight_column], dtype=float) + + sum_w = float(np.sum(w)) + sum_w2 = float(np.sum(w**2)) + sum_w2_e2 = float(np.sum((w**2) * (e1**2 + e2**2))) + + if mask_path is not None: + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask path not found: {mask_path}") + mask_candidate = mask_path + else: + mask_candidate = self.cc[ver].get("mask") + if isinstance(mask_candidate, str) and not os.path.isabs(mask_candidate): + mask_candidate = str(Path(self.cc[ver]["subdir"]) / mask_candidate) + if mask_candidate is not None and not os.path.exists(mask_candidate): + mask_candidate = None + + area_deg2 = None + if mask_candidate is not None and os.path.exists(mask_candidate): + area_deg2 = self._area_from_mask(mask_candidate) + elif cov_th.get("A") is not None: + area_deg2 = float(cov_th["A"]) + elif nside is not None: + area_deg2 = self._area_from_catalog(catalog_path, nside) + else: + raise ValueError( + f"Unable to determine survey area for {ver}. Provide mask_path or nside." + ) + + area_arcmin2 = area_deg2 * 3600.0 + + n_eff = (sum_w**2) / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 + sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 + + results = { + "area_deg2": area_deg2, + "n_eff": n_eff, + "sigma_e": sigma_e, + "sum_w": sum_w, + "sum_w2": sum_w2, + "catalog_size": n_rows, + } + + if overwrite_config: + if "cov_th" not in self.cc[ver]: + self.cc[ver]["cov_th"] = {} + self.cc[ver]["cov_th"]["A"] = float(area_deg2) + self.cc[ver]["cov_th"]["n_e"] = float(n_eff) + self.cc[ver]["cov_th"]["sigma_e"] = float(sigma_e) + self._write_catalog_config() + + return results + + def _area_from_catalog(self, catalog_path, nside): + data = fits.getdata(catalog_path, memmap=True) + ra = np.asarray(data["RA"], dtype=float) + dec = np.asarray(data["Dec"], dtype=float) + theta = np.radians(90.0 - dec) + phi = np.radians(ra) + pix = hp.ang2pix(nside, theta, phi, lonlat=False) + unique_pix = np.unique(pix) + return float(unique_pix.size * hp.nside2pixarea(nside, degrees=True)) + + def _area_from_mask(self, mask_map_path): + mask = hp.read_map(mask_map_path, dtype=np.float64) + return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) + + def _write_catalog_config(self): + with self.catalog_config_path.open("w") as file: + yaml.dump(self.cc, file, sort_keys=False) + def color_reset(self): print(colorama.Fore.BLACK, end="") @@ -239,6 +597,17 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise + def basename(self, version, treecorr_config=None, npatch=None): + cfg = treecorr_config or self.treecorr_config + patches = npatch or self.npatch + return ( + f"{version}_minsep={cfg['min_sep']}" + f"_maxsep={cfg['max_sep']}" + f"_nbins={cfg['nbins']}" + f"_npatch={patches}" + ) + + def calculate_rho_tau_stats(self): out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): @@ -246,13 +615,16 @@ def calculate_rho_tau_stats(self): self.print_start("Rho stats") for ver in self.versions: + base = self.basename(ver) rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( self.cc, ver, self.treecorr_config, out_dir, + base, method=self.cov_estimate_method, cov_rho=self.compute_cov_rho, + npatch=self.npatch, ) self.print_done("Rho stats finished") @@ -372,7 +744,9 @@ def calculate_ellipticity_dispersion(self): self._ellipticity_dispersion = ellipticity_dispersion def plot_rho_stats(self, abs=False): - filenames = [f"rho_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"rho_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "rho_stats.png" self.rho_stat_handler.plot_rho_stats( @@ -392,7 +766,9 @@ def plot_rho_stats(self, abs=False): ) def plot_tau_stats(self, plot_tau_m=False): - filenames = [f"tau_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"tau_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "tau_stats.png" self.tau_stat_handler.plot_tau_stats( @@ -472,9 +848,12 @@ def calculate_rho_tau_fits(self): self.cov_estimate_method, None ) + base = self.basename(ver) + flat_samples, result, q = get_samples( self.psf_fitter, ver, + base, cov_type=self.cov_estimate_method, apply_debias=npatch, sampler=self.rho_tau_method, @@ -484,7 +863,7 @@ def calculate_rho_tau_fits(self): self.rho_tau_fits["result_list"].append(result) self.rho_tau_fits["q_list"].append(q) - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] xi_psf_sys_samples = np.array([]).reshape(0, nbins) @@ -529,7 +908,7 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for i in range(100): self.psf_fitter.plot_xi_psf_sys( flat_sample[-i + 1], ver, color, alpha=0.1 @@ -579,7 +958,7 @@ def plot_rho_tau_fits(self): self.versions, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for yscale in ("linear", "log"): out_path = os.path.abspath( f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" @@ -1620,7 +1999,6 @@ def calculate_pure_eb( var_method="jackknife", cov_path_int=None, cosmo_cov=None, - redshift_file=None, n_samples=1000, ): """ @@ -1718,18 +2096,8 @@ def calculate_pure_eb( # Get redshift distribution if using analytic covariance if cov_path_int is not None: - if redshift_file is None: - try: - print("Inheriting redshift distribution from self.z_dist") - z_dist = self.z_dist - except AttributeError: - raise ValueError( - "redshift distribution must be provided either to this function" - " or to the CosmologyValidation class upon creation " - "if using an analytic covariance." - ) - else: - z_dist = np.loadtxt(redshift_file) + z, nz = self.get_redshift(version) + z_dist = np.column_stack([z, nz]) else: z_dist = None @@ -1762,7 +2130,6 @@ def plot_pure_eb( var_method="jackknife", cov_path_int=None, cosmo_cov=None, - redshift_file=None, n_samples=1000, results=None, **kwargs @@ -1800,8 +2167,6 @@ def plot_pure_eb( Path to integration covariance matrix for semi-analytical calculation cosmo_cov : pyccl.Cosmology, optional Cosmology for theoretical predictions in semi-analytical covariance - redshift_file : str, optional - Path to redshift distribution file for semi-analytical covariance n_samples : int Number of Monte Carlo samples for semi-analytical covariance (default: 1000) results : dict or list, optional @@ -1889,7 +2254,6 @@ def plot_pure_eb( var_method=var_method, cov_path_int=cov_path_int, cosmo_cov=cosmo_cov, - redshift_file=redshift_file, n_samples=n_samples, ) @@ -2117,7 +2481,6 @@ def plot_cosebis( plot_cosebis_covariance_matrix, plot_cosebis_modes, plot_cosebis_scale_cut_heatmap, - scale_cut_to_bins, ) # Use instance defaults if not specified @@ -2127,8 +2490,8 @@ def plot_cosebis( # Determine variance method based on whether theoretical covariance is used var_method = "analytic" if cov_path is not None else "jackknife" - - # Create output filename with integration parameters to match Snakemake expectations + + # Create output filename with integration parameters to match Snakemake out_stub = ( f"{output_dir}/{version}_cosebis_minsep={min_sep_int}_" f"maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}_" @@ -2166,7 +2529,9 @@ def plot_cosebis( # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use # full range if fiducial_scale_cut is not None: - plot_results = results[find_conservative_scale_cut_key(results, fiducial_scale_cut)] + plot_results = results[ + find_conservative_scale_cut_key(results, fiducial_scale_cut) + ] else: # Use full range result (largest scale cut) max_range_key = max(results.keys(), key=lambda x: x[1] - x[0]) @@ -2200,7 +2565,9 @@ def plot_cosebis( "max_sep": max_sep or self.treecorr_config["max_sep"], "nbins": nbins or self.treecorr_config["nbins"], } - gg_temp = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_temp) + gg_temp = self.calculate_2pcf( + version, npatch=npatch, **treecorr_config_temp + ) plot_cosebis_scale_cut_heatmap( results, @@ -2238,13 +2605,18 @@ def calculate_pseudo_cl_eb_cov(self): self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") - lmax = 2*self.nside - path_redshift_distr = self.data_base_dir + self.cc[ver]["shear"]["redshift_distr"] + lmax = 2 * self.nside + z, dndz = self.get_redshift(ver) + ell = np.arange(1, lmax + 1) pw = hp.pixwin(nside, lmax=lmax) + if pw.shape[0] != len(ell) + 1: + raise ValueError( + "Unexpected pixwin length for lmax=" + f"{lmax}: got {pw.shape[0]}, expected {len(ell)+1}" + ) + pw = pw[1:len(ell)+1] # Load redshift distribution and calculate theory C_ell - z, dndz = np.loadtxt(path_redshift_distr, unpack=True) - ell = np.arange(1, lmax + 1) fiducial_cl = ( get_theo_c_ell( ell=ell, diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 29ac84a..2ba9ee4 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -1,5 +1,6 @@ import os import time +from pathlib import Path import matplotlib.scale as mscale import matplotlib.ticker as ticker @@ -69,6 +70,9 @@ def get_transform(self): ] +def _extract_xip(correlations): + """Return flattened array of xip values from a list of correlations.""" + return np.array([corr.xip for corr in correlations]).flatten() def get_params_rho_tau(cat, survey="other"): # Set parameters @@ -111,42 +115,59 @@ def get_params_rho_tau(cat, survey="other"): return params -def get_rho_tau_w_cov(config, version, treecorr_config, outdir, method, cov_rho=False): - """ - Method to compute the covariance matrices of rho and tau-statistics of a given list of versions in cosmo_val. - Also computes rho and tau-statistics. - - Parameters - ---------- - versions : list - List of versions to compute the covariance matrices for. - method : str - Method to compute the covariance matrices. Options are 'jk' or 'th'. - """ +def get_rho_tau_w_cov( + config, + version, + treecorr_config, + outdir, + base, + method, + cov_rho=False, + npatch=None, +): + """Compute rho/tau statistics and, if requested, their covariance.""" if method == "th": nbin_ang, nbin_rad = 100, 200 rho_stat_handler, tau_stat_handler = get_rho_tau( - config, version, treecorr_config, outdir, cov_rho=cov_rho + config, + version, + treecorr_config, + outdir, + base, + cov_rho=cov_rho, ) get_theory_cov( config, version, treecorr_config, outdir, + base, nbin_ang=nbin_ang, nbin_rad=nbin_rad, ) return rho_stat_handler, tau_stat_handler elif method == "jk": - return get_jackknife_cov(config, version, treecorr_config, outdir) + return get_jackknife_cov( + config, + version, + treecorr_config, + outdir, + base, + npatch=npatch, + ) elif method == "sim": - if os.path.exists(outdir + "/cov_tau_" + version + "_th.npy"): - print( - f"Covariance from simulation available at the following file: {outdir+'/cov_tau_'+version+'_th.npy'}" - ) - print(f"Computing rho and tau statistics for the version: {version}") + tau_cov_path = Path(outdir) / f"cov_tau_{base}_th.npy" + + if tau_cov_path.exists(): + print(f"Found existing covariance at {tau_cov_path}") + print(f"Computing rho/tau statistics for {version}") return get_rho_tau( - config, version, treecorr_config, outdir, cov_rho=cov_rho + config, + version, + treecorr_config, + outdir, + base, + cov_rho=cov_rho, ) else: raise ValueError( @@ -156,7 +177,15 @@ def get_rho_tau_w_cov(config, version, treecorr_config, outdir, method, cov_rho= raise ValueError("Method must be either 'jk' or 'th' or 'sim'.") -def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): +def get_rho_tau( + config, + version, + treecorr_config, + outdir, + base, + cov_rho=False, + npatch=None, +): """ Compute rho and tau statistics for a given version of the catalogue. @@ -167,7 +196,7 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): version : str Version of the catalogue to use. treecorr_config : dict - Configuraion for treecorr. + TreeCorr configuration (must include 'min_sep', 'max_sep', and 'nbins'). outdir : str Output directory. """ @@ -177,16 +206,15 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): print("Compute Rho and Tau statistics for the version: ", version) start_time = time.time() - out_base = f"rho_stats_{version}.fits" - out_path = f"{outdir}/{out_base}" + rho_path = Path(outdir) / f"rho_stats_{base}.fits" rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=True ) - if os.path.exists(out_path): - print(f"Skipping rho statistics computation, file {out_path} already exists.") - rho_stat_handler.load_rho_stats(out_base) + if rho_path.exists(): + print(f"Skipping rho statistics computation, file {rho_path} already exists.") + rho_stat_handler.load_rho_stats(rho_path.name) else: rho_stat_handler.catalogs.set_params(params, outdir) @@ -208,20 +236,19 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): ) if cov_rho: - if not os.path.exists(outdir + "/cov_rho_" + version + ".npy"): - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() + cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + if not cov_rho_path.exists(): rho_stat_handler.compute_rho_stats( version, - out_base, + rho_path.name, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) else: - rho_stat_handler.compute_rho_stats(version, out_base, var_method=None) + rho_stat_handler.compute_rho_stats(version, rho_path.name, var_method=None) - out_base = f"tau_stats_{version}.fits" - out_path = f"{outdir}/{out_base}" + tau_path = Path(outdir) / f"tau_stats_{base}.fits" tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, @@ -230,9 +257,9 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): verbose=True, ) - if os.path.exists(out_path): - print(f"Skipping tau statistics computation, file {out_path} already exists.") - tau_stat_handler.load_tau_stats(out_base) + if tau_path.exists(): + print(f"Skipping tau statistics computation, file {tau_path} already exists.") + tau_stat_handler.load_tau_stats(tau_path.name) else: tau_stat_handler.catalogs.set_params(params, outdir) @@ -266,14 +293,20 @@ def get_rho_tau(config, version, treecorr_config, outdir, cov_rho=False): ) # function to extract the tau_+ - tau_stat_handler.compute_tau_stats(version, out_base, var_method=None) + tau_stat_handler.compute_tau_stats(version, tau_path.name, var_method=None) print(f"Time to compute rho and tau statistics: {time.time() - start_time:.2f} s") return rho_stat_handler, tau_stat_handler def get_theory_cov( - config, version, treecorr_config, outdir, nbin_ang=100, nbin_rad=100 + config, + version, + treecorr_config, + outdir, + base, + nbin_ang=100, + nbin_rad=100, ): """ Compute an analytical estimate of the covariance matrix of rho and tau-statistics. @@ -290,15 +323,17 @@ def get_theory_cov( path_psf = info["psf"]["path"] hdu_psf = info["psf"]["hdu"] - print("Computing the covariance matrix for the version: ", version) - start_time = time.time() + target_cov = Path(outdir) / f"cov_tau_{base}_th.npy" - if os.path.exists(outdir + "/cov_tau_" + version + "_th.npy"): + if target_cov.exists(): print( - f"Skipping covariance computation, file {outdir+'/cov_tau_'+version+'_th.npy'} already exists." + f"Skipping covariance computation, file {target_cov} already exists." ) return + print("Computing the covariance matrix for the version: ", version) + start_time = time.time() + cov_tau_th = CovTauTh( path_gal=path_gal, path_psf=path_psf, @@ -310,28 +345,39 @@ def get_theory_cov( params=params, ) - print( - "--- Computation of the rho and tau statistics for the covariance %s seconds ---" - % (time.time() - start_time) - ) + elapsed = time.time() - start_time + print(f"--- Rho/tau statistics for covariance computed in {elapsed:.2f}s ---") cov = cov_tau_th.build_cov(nbin_ang=nbin_ang, nbin_rad=nbin_rad) - print("--- Covariance computation %s seconds ---" % (time.time() - start_time)) - np.save(outdir + "/cov_tau_" + version + "_th.npy", cov) + print(f"--- Covariance matrix assembled in {time.time() - start_time:.2f}s ---") + target_cov.parent.mkdir(parents=True, exist_ok=True) + np.save(target_cov, cov) print("Saved covariance matrix of version: ", version) del cov_tau_th return -def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): +def get_jackknife_cov( + config, + version, + treecorr_config, + outdir, + base, + npatch, + ncov=100, +): """ Compute the covariance matrix of rho and tau-statistics using the jackknife method. Also compute rho and tau-statistics. """ - if os.path.exists(outdir + "/cov_tau_" + version + "_jk.npy"): + rho_filename = f"rho_stats_{base}.fits" + tau_filename = f"tau_stats_{base}.fits" + tau_cov_path = Path(outdir) / f"cov_tau_{base}_jk.npy" + + if tau_cov_path.exists(): print( - f"Skipping covariance computation, file {outdir+'/cov_tau_'+version+'_jk.npy'} already exists." + f"Skipping covariance computation, file {tau_cov_path} already exists." ) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False @@ -344,19 +390,22 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): verbose=True, ) + rho_path = Path(outdir) / rho_filename + tau_path = Path(outdir) / tau_filename + if rho_path.exists(): + rho_stat_handler.load_rho_stats(rho_path.name) + if tau_path.exists(): + tau_stat_handler.load_tau_stats(tau_path.name) return rho_stat_handler, tau_stat_handler - + params = get_params_rho_tau(config[version], survey=version) rho_stat_handler = RhoStat( output=outdir, treecorr_config=treecorr_config, verbose=False ) - out_base = f"rho_stats_{version}.fits" - rho_stat_handler.catalogs.set_params(params, outdir) - mask = version != "DES" square_size = params["square_size"] tau_stat_handler = TauStat( @@ -366,25 +415,18 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): verbose=True, ) - out_base = f"tau_stats_{version}.fits" - tau_stat_handler.catalogs.set_params(params, outdir) for i in range(ncov): - if not ( - os.path.exists(outdir + "/cov_tau_" + version + str(i) + ".npy") - and os.path.exists(outdir + "/cov_rho_" + version + str(i) + ".npy") - ): - + tau_chunk = outdir + f"/cov_tau_{version}{i}.npy" + rho_chunk = outdir + f"/cov_rho_{version}{i}.npy" + if not (os.path.exists(tau_chunk) and os.path.exists(rho_chunk)): print( - f"Computing rho-statistics of version {version} for jackknife patch {i+1}/{ncov}" + f"Computing rho-statistics for {version} (patch {i+1}/{ncov})" ) - if ( - f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict.keys() - ): - + if f"psf_{version}{i}" not in rho_stat_handler.catalogs.catalogs_dict: # Build catalogues rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], @@ -394,7 +436,9 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): hdu=config[version]["psf"]["hdu"], ) - tau_stat_handler.catalogs.catalogs_dict = rho_stat_handler.catalogs.catalogs_dict + tau_stat_handler.catalogs.catalogs_dict = ( + rho_stat_handler.catalogs.catalogs_dict + ) # Build the catalog of galaxies. PSF was computed above tau_stat_handler.build_cat_to_compute_tau( @@ -411,45 +455,48 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): ) npatch = rho_stat_handler.catalogs._params["patch_number"] - field = rho_stat_handler.catalogs.catalogs_dict[f"psf_{version}{i}"].getNField(max_top=int.bit_length(npatch)-1, coords='spherical') + field = rho_stat_handler.catalogs.catalogs_dict[ + f"psf_{version}{i}" + ].getNField(max_top=int.bit_length(npatch) - 1, coords="spherical") patch, centers = field.run_kmeans(npatch) #Update the patch centers of the catalogs for key, cat in rho_stat_handler.catalogs.catalogs_dict.items(): cat._centers = centers - field = cat.getNField(max_top=int.bit_length(npatch)-1, coords='spherical') + field = cat.getNField( + max_top=int.bit_length(npatch) - 1, coords="spherical" + ) cat._patch = field.kmeans_assign_patches(centers) - - - out_base = f"rho_stats_{version}.fits" # Compute and save rho stats - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() rho_stat_handler.compute_rho_stats( version + str(i), - out_base, + rho_filename, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) - out_base = f"tau_stats_{version}.fits" - # function to extract the tau_+ - only_p = lambda corrs: np.array([corr.xip for corr in corrs]).flatten() tau_stat_handler.compute_tau_stats( version + str(i), - out_base, + tau_filename, save_cov=True, - func=only_p, + func=_extract_xip, var_method="jackknife", ) #Update the keys in the dictionaries - rho_stat_handler.catalogs.catalogs_dict[f"psf_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_{version}{i}") - rho_stat_handler.catalogs.catalogs_dict[f"psf_error_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_error_{version}{i}") - rho_stat_handler.catalogs.catalogs_dict[f"psf_size_error_{version}{i+1}"] = rho_stat_handler.catalogs.catalogs_dict.pop(f"psf_size_error_{version}{i}") - tau_stat_handler.catalogs.catalogs_dict[f"gal_{version}{i+1}"] = tau_stat_handler.catalogs.catalogs_dict.pop(f"gal_{version}{i}") + rho_dict = rho_stat_handler.catalogs.catalogs_dict + tau_dict = tau_stat_handler.catalogs.catalogs_dict + rho_dict[f"psf_{version}{i+1}"] = rho_dict.pop(f"psf_{version}{i}") + rho_dict[f"psf_error_{version}{i+1}"] = rho_dict.pop( + f"psf_error_{version}{i}" + ) + rho_dict[f"psf_size_error_{version}{i+1}"] = rho_dict.pop( + f"psf_size_error_{version}{i}" + ) + tau_dict[f"gal_{version}{i+1}"] = tau_dict.pop(f"gal_{version}{i}") cov_tau_loc = np.zeros_like(np.load(outdir + f"/cov_tau_{version}0.npy")) cov_rho_loc = np.zeros_like(np.load(outdir + f"/cov_rho_{version}0.npy")) @@ -462,122 +509,162 @@ def get_jackknife_cov(config, version, treecorr_config, outdir, ncov=100): cov_tau = cov_tau_loc / ncov cov_rho = cov_rho_loc / ncov - np.save(outdir + "/cov_tau_" + version + "_jk.npy", cov_tau) - np.save(outdir + "/cov_rho_" + version + ".npy", cov_rho) + tau_cov_path.parent.mkdir(parents=True, exist_ok=True) + np.save(tau_cov_path, cov_tau) + + cov_rho_path = Path(outdir) / f"cov_rho_{base}_jk.npy" + cov_rho_path.parent.mkdir(parents=True, exist_ok=True) + np.save(cov_rho_path, cov_rho) return rho_stat_handler, tau_stat_handler -def get_samples(psf_fitter, version, cov_type="jk", apply_debias=None, sampler="emcee"): - """ - Samples (alpha, beta, eta) using the sampler 'emcee' or 'lsq' +def get_samples( + psf_fitter, + version, + base, + cov_type="jk", + apply_debias=None, + sampler="emcee", +): + """Return (alpha, beta, eta) samples using ``emcee`` or least squares. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance that provides ``load_*`` helpers. version : str - Version of the catalogue to use. - cov_type : str - Type of covariance matrix to use. Options are 'jk' or 'th' or 'sim'. - apply_debias : int - If not None, apply debiasing to the least square method. - sampler : str - Sampler to use. Options are 'emcee' or 'lsq'. (Default: 'emcee') + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename (e.g. ``SP_v1.4_minsep=…``) used for filenames. + cov_type : str, optional + Covariance label (``'jk'``, ``'th'``, or ``'sim'``). Defaults to ``'jk'``. + apply_debias : int or None, optional + Jackknife patch count used to debias samples. Disabled when ``None``. + sampler : str, optional + ``'emcee'`` for MCMC sampling, ``'lsq'`` for least squares + (default ``'emcee'``). """ if sampler == "emcee": return get_samples_emcee( - psf_fitter, version, cov_type=cov_type, apply_debias=apply_debias + psf_fitter, + version, + base, + cov_type=cov_type, + apply_debias=apply_debias, ) elif sampler == "lsq": return get_samples_lsq( - psf_fitter, version, cov_type=cov_type, apply_debias=apply_debias + psf_fitter, + version, + base, + cov_type=cov_type, + apply_debias=apply_debias, ) else: raise ValueError("Sampler must be either 'emcee' or 'lsq'.") def get_samples_emcee( - psf_fitter, version, nwalkers=124, nsamples=10000, cov_type="jk", apply_debias=None + psf_fitter, + version, + base, + nwalkers=124, + nsamples=10000, + cov_type="jk", + apply_debias=None, ): - """ - Samples (alpha, beta, eta) using the covariance of the tau statistics and emcee. + """Draw (alpha, beta, eta) samples using ``emcee`` and the tau covariance. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance managing rho/tau statistics and covariances. version : str - Version of the catalogue to use. - nwalkers : int - Number of walkers to use in the MCMC. (Default: 124) - nsamples : int - Number of samples to draw from the MCMC. (Default 10000) - cov_type : str - Type of covariance matrix to use. Options are 'jk' or 'th' or 'sim'. (Default: 'jk') + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename for locating statistics/covariance files. + nwalkers : int, optional + Number of walkers for the MCMC run (default ``124``). + nsamples : int, optional + Number of samples drawn per walker (default ``10000``). + cov_type : str, optional + Covariance label (``'jk'``/``'th'``/``'sim'``). Defaults to ``'jk'``. + apply_debias : int or None, optional + Jackknife patch count applied during debiasing. Disabled when ``None``. """ # Load rho and tau stats - psf_fitter.load_rho_stat("rho_stats_" + version + ".fits") - psf_fitter.load_tau_stat("tau_stats_" + version + ".fits") + psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") - # Check if the path exists - sample_file_path = psf_fitter.get_sample_file_path(version) + # Check if the path exists (use base for cache key to account for different TreeCorr configs) + sample_file_path = psf_fitter.get_sample_file_path(base) if os.path.exists(sample_file_path): - print(f"Skipping sample computation, file {sample_file_path} already exists.") - flat_samples = psf_fitter.load_samples(version) - mcmc_result, q = psf_fitter.get_mcmc_from_samples(version) + print(f"Skipping sampling; {sample_file_path} exists.") + flat_samples = psf_fitter.load_samples(base) + mcmc_result, q = psf_fitter.get_mcmc_from_samples(base) print(mcmc_result) # Or run MCMC else: print("MCMC sampling") - psf_fitter.load_covariance("cov_tau_" + version + "_" + cov_type + ".npy") + cov_filename = f"cov_tau_{base}_{cov_type}.npy" + psf_fitter.load_covariance(cov_filename, cov_type="tau") - npatch = apply_debias if (apply_debias is not None) else None + debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.run_chain( nwalkers=nwalkers, nsamples=nsamples, - npatch=npatch, - apply_debias=npatch is not None, + npatch=debias_npatch, + apply_debias=debias_npatch is not None, savefig="mcmc_samples_" + version + ".png", ) - psf_fitter.save_samples(flat_samples, version) + psf_fitter.save_samples(flat_samples, base) return flat_samples, mcmc_result, q -def get_samples_lsq(psf_fitter, version, apply_debias=None, cov_type="jk"): - """ - Samples (alpha, beta, eta) using the covariance of the tau statistics and least square method. +def get_samples_lsq( + psf_fitter, + version, + base, + apply_debias=None, + cov_type="jk", +): + """Compute least-squares samples of (alpha, beta, eta) using the tau covariance. Parameters ---------- psf_fitter : PSFFitter - Instance of PSFFitter. + PSF fitter instance managing rho/tau statistics and covariances. version : str - Version of the catalogue to use. - apply_debias : int - If not None, apply debiasing to the least square method. (Default: None) + Catalog identifier whose rho/tau statistics are sampled. + base : str + Precomputed basename for locating statistics/covariance files. + apply_debias : int or None, optional + Jackknife patch count applied during debiasing. Disabled when ``None``. + cov_type : str, optional + Covariance label (defaults to ``'jk'``). """ # Load rho and tau stats - psf_fitter.load_rho_stat("rho_stats_" + version + ".fits") - psf_fitter.load_tau_stat("tau_stats_" + version + ".fits") + psf_fitter.load_rho_stat(f"rho_stats_{base}.fits") + psf_fitter.load_tau_stat(f"tau_stats_{base}.fits") - # Check if the path exists - sample_file_path = psf_fitter.get_sample_path(version) + # Check if the path exists (use base for cache key to account for different TreeCorr configs) + sample_file_path = psf_fitter.get_sample_path(base) if os.path.exists(sample_file_path): - print(f"Skipping sample computation, file {sample_file_path} already exists.") - flat_samples = psf_fitter.load_samples(version) + print(f"Skipping sampling; {sample_file_path} exists.") + flat_samples = psf_fitter.load_samples(base) mcmc_result, q = psf_fitter.get_mcmc_from_samples(flat_samples) print(mcmc_result) # Or run MCMC else: print("Least square sampling") - psf_fitter.load_covariance( - "cov_tau_" + version + "_" + cov_type + ".npy", cov_type="tau" - ) - psf_fitter.load_covariance("cov_rho_" + version + ".npy", cov_type="rho") - npatch = apply_debias if (apply_debias is not None) else None + tau_covariance = f"cov_tau_{base}_{cov_type}.npy" + rho_covariance = f"cov_rho_{base}_jk.npy" + psf_fitter.load_covariance(tau_covariance, cov_type="tau") + psf_fitter.load_covariance(rho_covariance, cov_type="rho") + debias_npatch = apply_debias if (apply_debias is not None) else None flat_samples, mcmc_result, q = psf_fitter.get_least_squares_params_samples( - npatch=npatch, apply_debias=(npatch is not None) + npatch=debias_npatch, apply_debias=(debias_npatch is not None) ) - psf_fitter.save_samples(flat_samples, version) + psf_fitter.save_samples(flat_samples, base) return flat_samples, mcmc_result, q diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py new file mode 100644 index 0000000..afcd310 --- /dev/null +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -0,0 +1,323 @@ +"""UNIT TESTS FOR COSMOLOGY VALIDATION CLASS. + +This module contains integration tests for the CosmologyValidation class, +specifically testing the ellipticity_suffix parameter functionality for +handling leak-corrected ellipticity columns. + +:Author: cdaley + +""" + +import os +from collections import defaultdict +from pathlib import Path +from typing import Dict, Iterator, Tuple + +import pytest +import yaml + +from sp_validation.cosmo_val import CosmologyValidation + + +class TestCosmologyValidation: + """Test CosmologyValidation initialization and additive bias calculation.""" + + @pytest.fixture + def base_config(self, tmp_path): + """Common configuration parameters for tests.""" + # Get the path to the repo root (3 levels up from this test file) + repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + ) + catalog_config = os.path.join( + repo_root, "notebooks", "cosmo_val", "cat_config.yaml" + ) + + # Use temporary directory for outputs + output_dir = tmp_path / "test_output" + output_dir.mkdir() + + return { + "catalog_config": catalog_config, + "output_dir": str(output_dir), + "npatch": 1, + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + } + + @staticmethod + def _make_seed_config(tmp_path, shear_filename): + """Create a minimal catalog config for seed variant testing.""" + base_version = "TestCatalog" + base_dir = tmp_path / "catalog" + base_dir.mkdir() + (base_dir / shear_filename).touch() + + star_filename = "star_seed_1234.fits" + (base_dir / star_filename).touch() + + nz_dir = tmp_path / "nz" + nz_dir.mkdir() + (nz_dir / "dndz.txt").write_text("0.1 1.0\n") + + output_dir = tmp_path / "output" + output_dir.mkdir() + + config_path = tmp_path / "seed_config.yaml" + config_data = { + "nz": { + "subdir": str(nz_dir), + "dndz": {"blind": "A", "path": "dndz.txt"}, + }, + "paths": {"output": str(output_dir)}, + base_version: { + "subdir": str(base_dir), + "pipeline": "SP", + "shear": { + "path": shear_filename, + "w_col": "w", + "e1_col": "e1", + "e2_col": "e2", + "e1_col_corrected": "e1_corr", + "e2_col_corrected": "e2_corr", + }, + "star": {"path": star_filename}, + }, + } + config_path.write_text(yaml.dump(config_data, sort_keys=False)) + + params = { + "catalog_config": str(config_path), + "output_dir": str(output_dir), + "npatch": 1, + "theta_min": 1.0, + "theta_max": 250.0, + "nbins": 20, + } + return params, base_version + + def test_additive_bias_base_columns(self, base_config): + """Test additive bias calculation using base ellipticity columns. + + This test initializes CosmologyValidation without an ellipticity_suffix, + which means it will use the default columns defined in the catalog + configuration. Tests SP_v1.4.5 with full additive bias computation. + """ + version = "SP_v1.4.5" + e1_col = "e1" + e2_col = "e2" + + cv = CosmologyValidation( + versions=[version], + **base_config, + ) + + # Verify version names remain unchanged + assert cv.versions == [version] + + # Verify the ellipticity columns are the base columns + assert cv.cc[version]["shear"]["e1_col"] == e1_col + assert cv.cc[version]["shear"]["e2_col"] == e2_col + + # Calculate additive bias + cv.calculate_additive_bias() + + # Verify c1 and c2 were calculated and stored + assert hasattr(cv, "_c1") and hasattr(cv, "_c2") + assert version in cv.c1 + assert version in cv.c2 + + # Verify the values are numeric (not NaN or None) + assert isinstance(cv.c1[version], float) + assert isinstance(cv.c2[version], float) + + def test_additive_bias_leak_corrected_columns(self, base_config): + """Test additive bias calculation using leak-corrected columns. + + This test requests a leak-corrected version by passing "SP_v1.4.6_leak_corr" + as the version name. The CosmologyValidation class automatically detects + the _leak_corr suffix and creates the config entry using e1_col_corrected + and e2_col_corrected from the base version. + """ + base_version = "SP_v1.4.6" + version_leak_corr = f"{base_version}_leak_corr" + + cv = CosmologyValidation( + versions=[version_leak_corr], + **base_config, + ) + + # Verify version names include the _leak_corr suffix + assert cv.versions == [version_leak_corr] + + # Verify the leak-corrected config was auto-created with corrected columns + assert cv.cc[version_leak_corr]["shear"]["e1_col"] == "e1_leak_corrected" + assert cv.cc[version_leak_corr]["shear"]["e2_col"] == "e2_leak_corrected" + + # Verify original config entry remains unchanged + assert cv.cc[base_version]["shear"]["e1_col"] == "e1" + assert cv.cc[base_version]["shear"]["e2_col"] == "e2" + + # Calculate additive bias + cv.calculate_additive_bias() + + # Verify c1 and c2 were calculated and stored with leak-corrected name + assert hasattr(cv, "_c1") and hasattr(cv, "_c2") + assert version_leak_corr in cv.c1 + assert version_leak_corr in cv.c2 + + # Verify the values are numeric + assert isinstance(cv.c1[version_leak_corr], float) + assert isinstance(cv.c2[version_leak_corr], float) + + @staticmethod + def _iter_catalog_entries(config: Dict[str, Dict]) -> Iterator[Tuple[str, Dict]]: + """Yield (name, entry) pairs for catalog-like entries in the config.""" + for name, entry in config.items(): + if not isinstance(entry, dict): + continue + if "subdir" not in entry: + continue + yield name, entry + + @staticmethod + def _resolve(base: Path, candidate: str) -> Path: + """Return an absolute path given a base directory and a candidate string.""" + candidate_path = Path(candidate) + return candidate_path if candidate_path.is_absolute() else base / candidate_path + + def test_catalog_paths_exist(self, base_config): + """Verify that catalog paths for active versions exist on disk. + + This is a lightweight test that checks that all files referenced in the + catalog configuration for UNIONS analysis versions actually exist. It + discovers versions programmatically from cat_config.yaml rather than + using hardcoded lists. + """ + # Get the path to catalog config + repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + ) + catalog_config_path = os.path.join( + repo_root, "notebooks", "cosmo_val", "cat_config.yaml" + ) + + config = yaml.safe_load(Path(catalog_config_path).read_text()) + + missing = defaultdict(set) + working = [] + nonfunctional = defaultdict(set) + + for version, entry in self._iter_catalog_entries(config): + # Skip nz entries and versions already tested in heavy tests + if version == "nz": + continue + + base = Path(entry["subdir"]) + version_missing = set() + + # Check shear, star, and psf files + for block_name in ("shear", "star", "psf"): + block = entry.get(block_name) + if not block: + continue + resolved_path = self._resolve(base, block["path"]) + if not resolved_path.is_file(): + version_missing.add(block_name) + + if version_missing: + nonfunctional[version] = version_missing + else: + working.append(version) + + # Print summary + print(f"\n✓ Working versions ({len(working)}):") + for v in sorted(working): + print(f" - {v}") + + if nonfunctional: + print(f"\n✗ Non-functional versions ({len(nonfunctional)}):") + for v in sorted(nonfunctional.keys()): + print(f" - {v}: missing {nonfunctional[v]}") + + assert not nonfunctional, ( + "Catalog configuration references missing files: " + f"{dict(nonfunctional)}" + ) + + def test_seed_variant_updates_shear_path(self, tmp_path): + """Seeded versions should materialize a seed-specific shear path.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + seed_version = f"{base_version}_seed007" + + cv = CosmologyValidation(versions=[seed_version], **params) + + assert cv.versions == [seed_version] + assert seed_version in cv.cc + assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + + def test_seed_leak_corr_materializes_seed_first(self, tmp_path): + """_seed_leak_corr should clone the seed variant before leak fixes.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_seed_1234.fits" + ) + leak_version = f"{base_version}_seed007_leak_corr" + seed_version = f"{base_version}_seed007" + + cv = CosmologyValidation(versions=[leak_version], **params) + + assert cv.versions == [leak_version] + assert seed_version in cv.cc + assert cv.cc[seed_version]["shear"]["path"].endswith("shear_seed_007.fits") + assert cv.cc[leak_version]["shear"]["e1_col"] == "e1_corr" + assert cv.cc[leak_version]["shear"]["e2_col"] == "e2_corr" + + def test_seed_variant_without_token_errors(self, tmp_path): + """Missing seed token in shear path should raise a descriptive error.""" + params, base_version = self._make_seed_config( + tmp_path, shear_filename="shear_base.fits" + ) + seed_version = f"{base_version}_seed123" + + with pytest.raises(ValueError, match="seed"): + CosmologyValidation(versions=[seed_version], **params) + + def test_v1_4_6_glass_mock_seed_variant(self, base_config): + """Test that v1.4.6 glass mock seed variant loads with correct path.""" + seed = 9 + seed_version = f"SP_v1.4.6_glass_mock_seed{seed}" + + cv = CosmologyValidation( + versions=[seed_version], + **base_config, + ) + + # Verify version was created + assert cv.versions == [seed_version] + assert seed_version in cv.cc + + # Verify seed was substituted in shear path + expected_filename = f"unions_glass_sim_{seed:05d}_4096.fits" + assert expected_filename in cv.cc[seed_version]["shear"]["path"] + + # Verify path points to v1.4.6 glass mock directory + assert "glass_mock_v1.4.6" in cv.cc[seed_version]["shear"]["path"] + + def test_v1_4_6_glass_mock_default_seed(self, base_config): + """Test that glass mock without seed suffix uses the default seed_00001.""" + cv = CosmologyValidation( + versions=["SP_v1.4.6_glass_mock"], + **base_config, + ) + + # Verify version loads without seed suffix + assert cv.versions == ["SP_v1.4.6_glass_mock"] + assert "SP_v1.4.6_glass_mock" in cv.cc + + # Verify it uses the default path (seed_00001 for v1.4.6) + path = cv.cc["SP_v1.4.6_glass_mock"]["shear"]["path"] + assert "unions_glass_sim_00001_4096.fits" in path + assert "glass_mock_v1.4.6" in path