-band magnitude histograms

config/calibration/mask_v1.X.6.yaml

@@ -31,13 +31,13 @@ dat:
 
   # Number of epochs
   - col_name: N_EPOCH
-    label: r"$n_{\rm epoch}$"
+    label: $n_{\rm epoch}$
     kind: greater_equal
     value: 2
 
   # Magnitude range
   - col_name: mag
-    label: mag range
+    label: r
     kind: range
     value: [15, 30]
 
@@ -86,7 +86,7 @@ dat_ext:
 
   # Rough pointing coverage
   - col_name: npoint3
-    label: r"$n_{\rm pointing}$"
+    label: $n_{\rm pointing}$
     kind: greater_equal
     value: 3
 

notebooks/cosmo_val/cat_config.yaml

@@ -616,6 +616,8 @@ SP_v1.4.6:
     covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
     path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
     redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
+    ra_col: RA
+    dec_col: Dec
     w_col: w_des
     e1_col: e1
     e1_col_corrected: e1_leak_corrected
@@ -629,6 +631,7 @@ SP_v1.4.6:
     e1_col: e1
     e2_col: e2
     path: unions_shapepipe_star_2024_v1.4.a.fits
+  patch_number: 100
 SP_v1.4.7:
   subdir: /n17data/UNIONS/WL/v1.4.x
   pipeline: SP
@@ -661,6 +664,8 @@ SP_v1.4.7:
     covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
     path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits
     redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
+    ra_col: RA
+    dec_col: Dec
     w_col: w_des
     e1_col: e1
     e1_col_corrected: e1_leak_corrected
@@ -706,6 +711,8 @@ SP_v1.4.8:
     covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
     path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits
     redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
+    ra_col: RA
+    dec_col: Dec
     w_col: w_des
     e1_col: e1
     e1_col_corrected: e1_leak_corrected
@@ -719,6 +726,110 @@ SP_v1.4.8:
     e1_col: e1
     e2_col: e2
     path: unions_shapepipe_star_2024_v1.4.a.fits
+
+# For additive bias only
+SP_v1.4.6_uncal:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_des
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.6_uncal_w_iv:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_iv
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.6_uncal_w_1:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: one
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.5_uncal:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_des
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.7_uncal:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_des
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.8_uncal:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_des
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
 SP_v1.4_LFmask_8k:
   subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask
   pipeline: SP

notebooks/cosmo_val/catalog_paper_plot/check_gaia.ipynb

@@ -0,0 +1,73 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "8049856e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from matplotlib import pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d7f7568b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from astropy.io import fits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "62854cbc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/automnt/n17data/mkilbing/astro/Runs/flexion/simuls_MAB\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4c311b19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cat = fits.getdata(\"gaia2.fits\", 1)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/cosmo_val/catalog_paper_plot/download_gaia_catalogues.py

@@ -0,0 +1,100 @@
+# %%
+# Calibrate comprehensive catalogue
+
+# %%
+from IPython import get_ipython
+
+# %%
+# enable autoreload for interactive sessions
+ipython = get_ipython()
+if ipython is not None:
+    ipython.run_line_magic("reload_ext", "autoreload")
+    ipython.run_line_magic("autoreload", "2")
+    ipython.run_line_magic("reload_ext", "log_cell_time")
+
+import numpy as np
+import time
+from astroquery.gaia import Gaia
+import astropy.units
+from astropy.coordinates import SkyCoord
+from textwrap import dedent
+
+
+def get_bins(results, key="phot_g_mean_mag", n_bins=3):
+
+    quantiles_raw = np.percentile(results[key], np.linspace(0, 100, n_bins + 1))
+    # round
+    quantiles = np.round(quantiles_raw, 1)
+
+    print("Magnitude bin edges:")
+    subsets = []
+    for i in range(n_bins):
+        this_subset = results[
+            (results[key] >= quantiles[i]) & (results[key] < quantiles[i + 1])
+        ]
+        subsets.append(this_subset)
+
+        print(
+            f"  Bin {i+1}: {quantiles[i]:.2f}, {quantiles[i+1]:.2f}, N={len(this_subset)}"
+        )
+
+    return subsets, quantiles
+
+
+def do_query(do_async=True, nmax=3_000_000):
+
+    # Define query
+    query = dedent(
+        f"""
+        SELECT TOP {nmax} ra, dec, phot_g_mean_mag, ruwe
+		FROM gaiadr3.gaia_source
+		WHERE dec > 30
+			AND dec < 75
+			AND phot_g_mean_mag < 20
+    		AND phot_g_mean_mag IS NOT NULL
+            AND ruwe < 1.4
+        ORDER BY random_index
+	"""
+    ).strip()
+
+
+    # Launch the query
+    if not do_async:
+        print("Retrieveing GAIA data (sync)")
+        job = Gaia.launch_job(query)
+    else:
+        print("Retrieveing GAIA data (async)")
+        job = Gaia.launch_job_async(query)
+
+        while not job.is_finished():
+            print(f"Job status: {job.get_phase()}")
+            time.sleep(10)
+
+    results = job.get_results()
+
+    print(f"Retrieved {len(results)} sources")
+    print(results[:10])  # Show first 10 results
+
+    return results
+
+
+def write_subsets(subsets, quantiles):
+
+    for idx, subset in enumerate(subsets):
+
+        if idx == 0:
+            out_path = f"gaia_stars_g_smaller_{quantiles[idx + 1]}.fits"
+        elif idx == len(subsets) - 1:
+            out_path = f"gaia_stars_g_larger_{quantiles[idx]}.fits"
+        else:
+            out_path = f"gaia_stars_g_in_{quantiles[idx]}_{quantiles[idx + 1]}.fits"
+        print(f"Writing {len(subset)} objects to {out_path}")
+        subset.write(out_path, format="fits", overwrite=True)
+
+
+# %% Main program
+results = do_query(do_async=True)
+
+subsets, quantiles = get_bins(results)
+
+write_subsets(subsets, quantiles)