From b3e9225635d24759c8e9446d71c58ef53e92ef6d Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 10:51:11 -0700 Subject: [PATCH 01/22] Ignore local venv directories Co-Authored-By: Claude Sonnet 5 --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 3f7c7834..2ad0eb0e 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,10 @@ _build/ #pycache __pycache__ +# Local Python virtual environments +.venv/ +venv/ + # Local working notes, not part of the tutorials CLAUDE.md OU_PR_REVIEW.md From 562c35d92d4ff99264368be889b37b421bbdc3bb Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 10:51:11 -0700 Subject: [PATCH 02/22] Point spherex_cutouts at irsatest and filter to DR1 Co-Authored-By: Claude Sonnet 5 --- tutorials/spherex/spherex_cutouts.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tutorials/spherex/spherex_cutouts.md b/tutorials/spherex/spherex_cutouts.md index ce50448b..8b563aab 100644 --- a/tutorials/spherex/spherex_cutouts.md +++ b/tutorials/spherex/spherex_cutouts.md @@ -104,22 +104,29 @@ Each row in the table corresponds to a single cutout and includes the data acces The results are sorted from oldest to newest. ```{code-cell} ipython3 +# TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and +# revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +IRSA_HOST = "https://irsatest.ipac.caltech.edu" + # Define the service endpoint for IRSA's Table Access Protocol (TAP) # so that we can query SPHEREx metadata tables. -service = pyvo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") +service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") # Define a query that will search the appropriate SPHEREx metadata tables # for spectral images that cover the chosen coordinates and match the # specified bandpass. Return the cutout data access URL and the time of observation. # Sort by observation time. +# spherex.plane.datarelease is a date, not a release name, so DR1 rows are +# selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - 'https://irsa.ipac.caltech.edu/' || a.uri || '?center={ra.value},{dec.value}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={ra.value},{dec.value}d&size={size.value}' AS uri, p.time_bounds_lower FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid WHERE 1 = CONTAINS(POINT('ICRS', {ra.value}, {dec.value}), p.poly) AND p.energy_bandpassname = '{bandpass}' + AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ @@ -166,7 +173,7 @@ def process_cutout(row, ra, dec, cache): with fits.open(row["uri"], cache=cache) as hdulist: # There are seven HDUs: # 0 contains minimal metadata in the header and no data. - # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, PSF, WCS-WAVE + # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE header = hdulist[1].header # Compute pixel coordinates corresponding to cutout position. From f12b0e287df44d4ef51171e9255a7b2ec9c191af Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 10:51:14 -0700 Subject: [PATCH 03/22] Point spherex_psf at irsatest, filter to DR1, fix ePSF fallback guard Co-Authored-By: Claude Sonnet 5 --- tutorials/spherex/spherex_psf.md | 71 ++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/tutorials/spherex/spherex_psf.md b/tutorials/spherex/spherex_psf.md index c96188f1..b7bb3ece 100644 --- a/tutorials/spherex/spherex_psf.md +++ b/tutorials/spherex/spherex_psf.md @@ -112,21 +112,28 @@ size = 0.01 * u.degree Once we defined the coordinates of interest and the size of the cutout, we run a TAP query to gather all SPHEREx spectral images that cover the coordinates. ```{code-cell} ipython3 +# TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and +# revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +IRSA_HOST = "https://irsatest.ipac.caltech.edu" + # Define the service endpoint for IRSA's Table Access Protocol (TAP) # so that we can query SPHEREx metadata tables. -service = pyvo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") +service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") # Define a query that will search the appropriate SPHEREx metadata tables # for spectral images that cover the chosen coordinates of interest. # Return the cutout data access URL and the time of observation. # Sort by observation time. +# spherex.plane.datarelease is a date, not a release name, so DR1 rows are +# selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - 'https://irsa.ipac.caltech.edu/' || a.uri || '?center={ra.value},{dec.value}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={ra.value},{dec.value}d&size={size.value}' AS uri, p.time_bounds_lower, p.energy_bandpassname, p.time_bounds_lower, p.time_bounds_upper, p.provenance_version FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid WHERE 1 = CONTAINS(POINT('ICRS', {ra.value}, {dec.value}), p.poly) + AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ @@ -180,7 +187,7 @@ for attempt in range(max_retries): hdul.info() ``` -The downloaded SPHEREx image cutout contains 5 FITS layers, which are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +The downloaded SPHEREx image cutout contains a primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE), which are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). Let's also get the version of the image. The version is needed later to decide how to handle the PSF. For obtaining the version of the image, we define a handy function. @@ -267,12 +274,14 @@ Note that the ePSFs are stored in the form of a binary table, which makes it str try: epsf_hdu = image_hdul["EPSF"] except KeyError: + epsf_in_image = False print( "This image does not contain the EPSF extension. " "It was likely created with an older version. " - "Please proceed to Section 5.2." + "Proceeding to Section 5.2." ) else: + epsf_in_image = True epsf_bintable = Table(epsf_hdu.data) epsf_header = epsf_hdu.header print("ePSF loaded.") @@ -290,8 +299,9 @@ Eventually, after the reprocessing of the data products, the older version LVF i For this, we first have to figure out the detector on which the image was taken, because the PSF depends on the wavelength (hence detector). ```{code-cell} ipython3 -this_detector = image_hdul['IMAGE'].header["DETECTOR"] -print(f"Detector is {this_detector}") +if not epsf_in_image: + this_detector = image_hdul['IMAGE'].header["DETECTOR"] + print(f"Detector is {this_detector}") ``` Next, we load the calibration file for that detector. Note that each detector has one ePSF calibration file as for now we do not assume that the calibration changes as a function of time. @@ -303,37 +313,44 @@ Specifically, the ePSF calibration files are stored in the respective release di We first construct a dictionary linking the detectors with the most current ePSFs in this bucket. ```{code-cell} ipython3 -ePSF_calibs = { - "1": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/1/epsf_D1_spx_cal-epsf-v1-2026-191.fits", - "2": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/2/epsf_D2_spx_cal-epsf-v1-2026-191.fits", - "3": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v2-2026-191/3/epsf_D3_spx_cal-epsf-v2-2026-191.fits", - "4": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/4/epsf_D4_spx_cal-epsf-v1-2026-191.fits", - "5": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/5/epsf_D5_spx_cal-epsf-v1-2026-191.fits", - "6": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/6/epsf_D6_spx_cal-epsf-v1-2026-191.fits" -} +if not epsf_in_image: + ePSF_calibs = { + "1": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/1/epsf_D1_spx_cal-epsf-v1-2026-191.fits", + "2": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/2/epsf_D2_spx_cal-epsf-v1-2026-191.fits", + "3": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v2-2026-191/3/epsf_D3_spx_cal-epsf-v2-2026-191.fits", + "4": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/4/epsf_D4_spx_cal-epsf-v1-2026-191.fits", + "5": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/5/epsf_D5_spx_cal-epsf-v1-2026-191.fits", + "6": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/6/epsf_D6_spx_cal-epsf-v1-2026-191.fits" + } ``` -Now we can rad in the correct calibration product ePSF corresponding to the detector in which the image was taken. +Now we can read in the correct calibration product ePSF corresponding to the detector in which the image was taken. ```{code-cell} ipython3 -with fits.open(ePSF_calibs[str(this_detector)]) as hdul: - hdul.info() - epsf_bintable = Table(hdul['EPSF'].data) - epsf_header = hdul['EPSF'].header - epsf_mosaic = hdul['EPSF_MOSAIC'].data +if not epsf_in_image: + with fits.open(ePSF_calibs[str(this_detector)]) as hdul: + hdul.info() + epsf_bintable = Table(hdul['EPSF'].data) + epsf_header = hdul['EPSF'].header + epsf_mosaic = hdul['EPSF_MOSAIC'].data +else: + print("ePSF was already loaded from the image in Section 5.1; skipping the calibration-file fallback.") ``` The calibration product contains the same `EPSF` extension as the SPHEREx LVF image files. In addition, the calibration products contains an `EPSF_MOSAIC` layer, which is *not* available in the LVF images. The ePSF mosaic allows a quick visualization of all the ePSFs on the detector. ```{code-cell} ipython3 -fig = plt.figure(figsize=(16,10)) -ax1 = fig.add_subplot(1,2,1) - -im1 = ax1.imshow(epsf_mosaic, origin='lower') -ax1.set_xlabel("$x$") -ax1.set_ylabel("$y$") -plt.show() +if not epsf_in_image: + fig = plt.figure(figsize=(16,10)) + ax1 = fig.add_subplot(1,2,1) + + im1 = ax1.imshow(epsf_mosaic, origin='lower') + ax1.set_xlabel("$x$") + ax1.set_ylabel("$y$") + plt.show() +else: + print("EPSF_MOSAIC is only available in the calibration product (Section 5.2), not in the LVF image itself.") ``` ```{note} From 2500fc4085eae5be39c486ab36fff7bc51a24b79 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 13:26:00 -0700 Subject: [PATCH 04/22] Update Explanatory Supplement link for DR1 naming Co-Authored-By: Claude Sonnet 5 --- tutorials/spherex/spherex_cutouts.md | 2 +- tutorials/spherex/spherex_psf.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorials/spherex/spherex_cutouts.md b/tutorials/spherex/spherex_cutouts.md index 8b563aab..64c11653 100644 --- a/tutorials/spherex/spherex_cutouts.md +++ b/tutorials/spherex/spherex_cutouts.md @@ -38,7 +38,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). ## 3. Imports diff --git a/tutorials/spherex/spherex_psf.md b/tutorials/spherex/spherex_psf.md index b7bb3ece..20430ede 100644 --- a/tutorials/spherex/spherex_psf.md +++ b/tutorials/spherex/spherex_psf.md @@ -48,7 +48,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ @@ -187,7 +187,7 @@ for attempt in range(max_retries): hdul.info() ``` -The downloaded SPHEREx image cutout contains a primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE), which are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +The downloaded SPHEREx image cutout contains a primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE), which are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). Let's also get the version of the image. The version is needed later to decide how to handle the PSF. For obtaining the version of the image, we define a handy function. From 818fea21ff80b3f61e02143f7837638f189c38fd Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 13:35:22 -0700 Subject: [PATCH 05/22] Update Explanatory Supplement link for DR1 naming Co-Authored-By: Claude Sonnet 5 --- tutorials/spherex/spherex_mosaic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/spherex/spherex_mosaic.md b/tutorials/spherex/spherex_mosaic.md index 3e3a451a..08c68a3c 100644 --- a/tutorials/spherex/spherex_mosaic.md +++ b/tutorials/spherex/spherex_mosaic.md @@ -42,7 +42,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ From 54b40bfda99b4d2c6a861243d9ef846ecca55117 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 13:35:22 -0700 Subject: [PATCH 06/22] Mention periodic Data Releases on SPHEREx landing page Co-Authored-By: Claude Sonnet 5 --- tutorials/spherex/spherex.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/spherex/spherex.md b/tutorials/spherex/spherex.md index 1f9ea93b..774070ca 100644 --- a/tutorials/spherex/spherex.md +++ b/tutorials/spherex/spherex.md @@ -4,7 +4,7 @@ SPHEREx observes the sky from roughly 0.75–5.0 µm using a single instrument that provides low-resolution spectroscopy (R ≈ 40–150) in hundreds of spectral channels for every point on the sky. Its science goals span cosmology, galaxy evolution, and the interstellar medium, enabling measurements of large-scale structure, the cosmic history of star formation, and the distribution of key molecules and ices in the Milky Way and nearby galaxies. -SPHEREx data releases include weekly [Quick Release spectral image products](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-products/) (multi-extension FITS files containing calibrated near-infrared surface brightness, variance, flags, modeled backgrounds, PSFs, and wavelength WCS) along with ancillary calibration and metadata files such as gain matrices, dark current maps, solid angle pixel maps, and detailed spectral WCS products for each detector. +SPHEREx data releases include weekly [Quick Release spectral image products](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-products/) as well as periodic Data Releases (e.g., DR1) (multi-extension FITS files containing calibrated near-infrared surface brightness, variance, flags, modeled backgrounds, PSFs, and wavelength WCS) along with ancillary calibration and metadata files such as gain matrices, dark current maps, solid angle pixel maps, and detailed spectral WCS products for each detector. ```{notebook-gallery} notebook_metadata.yml tutorials/spherex/spherex_intro.md From db5747e20de3187408348bdc60e3a569dddf7191 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 13:35:22 -0700 Subject: [PATCH 07/22] Point SDT TAP query at irsatest and filter to DR1 Co-Authored-By: Claude Sonnet 5 --- .../spherex_source_discovery_tool_demo.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index dd7df3f5..41cbd3b3 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -47,7 +47,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ @@ -649,9 +649,13 @@ Note that the following procedure follows the one shown in the SPHEREx [Image Cu ``` ```{code-cell} ipython3 +# TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and +# revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +IRSA_HOST = "https://irsatest.ipac.caltech.edu" + # Define the service endpoint for IRSA's Table Access Protocol (TAP) # so that we can query SPHEREx metadata tables. -service = pyvo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") +service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") # Specify cutout size (approximately 60 pixels on a side) size = 0.1 * u.degree @@ -660,13 +664,16 @@ size = 0.1 * u.degree # for spectral images that cover the chosen coordinates and match the # specified bandpass. Return the cutout data access URL and the time of observation. # Sort by observation time. +# spherex.plane.datarelease is a date, not a release name, so DR1 rows are +# selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - 'https://irsa.ipac.caltech.edu/' || a.uri || '?center={matched_stars['ra'][0]},{matched_stars['dec'][0]}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={matched_stars['ra'][0]},{matched_stars['dec'][0]}d&size={size.value}' AS uri, p.time_bounds_lower FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid WHERE 1 = CONTAINS(POINT('ICRS', {matched_stars['ra'][0]}, {matched_stars['dec'][0]}), p.poly) + AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ @@ -704,7 +711,7 @@ def process_cutout(row, ra, dec, cache, retries=3, delay=5): with fits.open(row["uri"]) as hdulist: # There are seven HDUs: # 0 contains minimal metadata in the header and no data. - # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, PSF, WCS-WAVE + # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE header = hdulist[1].header # Fetch spatial and spectral WCS From 5c16dcfb478ccf82bf3087a7001ed6b43b2992a4 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 14:26:15 -0700 Subject: [PATCH 08/22] Fold in #150 refactor and #238 header preservation fix Co-Authored-By: Claude Sonnet 5 --- tutorials/spherex/spherex_cutouts.md | 85 ++++++++++++++++------------ tutorials/spherex/spherex_psf.md | 34 ++++++----- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/tutorials/spherex/spherex_cutouts.md b/tutorials/spherex/spherex_cutouts.md index 64c11653..2af6ad95 100644 --- a/tutorials/spherex/spherex_cutouts.md +++ b/tutorials/spherex/spherex_cutouts.md @@ -46,7 +46,7 @@ The following packages must be installed to run this notebook. ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed. -# !pip install numpy astropy pyvo matplotlib +# !pip install numpy astropy astroquery matplotlib ``` ```{code-cell} ipython3 @@ -58,11 +58,12 @@ import urllib.error import astropy.units as u import matplotlib.pyplot as plt import numpy as np -import pyvo from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.table import Table +from astropy.visualization import quantity_support from astropy.wcs import WCS +from astroquery.ipac.irsa import Irsa # Suppress logging temporarily to prevent astropy # from repeatedly printing out warning notices related to alternate WCSs @@ -73,6 +74,9 @@ logging.getLogger('astropy').setLevel(logging.ERROR) # astropy's default timeout limit. Increase it. from astropy.utils.data import conf conf.remote_timeout = 120 + +# Let matplotlib recognize astropy Quantities and auto-label axes with their units. +quantity_support() ``` ## 4. Specify inputs and outputs @@ -83,8 +87,7 @@ In this example, we are creating cutouts of the Pinwheel galaxy (M101) for the S ```{code-cell} ipython3 # Choose a position. -ra = 210.80227 * u.degree -dec = 54.34895 * u.degree +coord = SkyCoord(ra=210.80227 * u.degree, dec=54.34895 * u.degree, frame="icrs") # Choose a cutout size. size = 0.1 * u.degree @@ -98,7 +101,7 @@ output_filename = 'spherex_cutouts_mef.fits' ## 5. Query IRSA for a list of cutouts that satisfy the criteria specified above. -Here we show how to use the `pyvo` TAP SQL query to retrieve all images that overlap with the position defined above. +Here we show how to use `astroquery`'s IRSA TAP interface to retrieve all images that overlap with the position defined above. This query will retrieve a table of URLs that link to the MEF cutouts. Each row in the table corresponds to a single cutout and includes the data access URL and an observation timestamp. The results are sorted from oldest to newest. @@ -106,11 +109,10 @@ The results are sorted from oldest to newest. ```{code-cell} ipython3 # TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and # revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +# Note: `Irsa.tap_url` must be set directly -- `astroquery.ipac.irsa.conf.tap_url` is only read +# once when the module is first imported, so changing it afterward has no effect. IRSA_HOST = "https://irsatest.ipac.caltech.edu" - -# Define the service endpoint for IRSA's Table Access Protocol (TAP) -# so that we can query SPHEREx metadata tables. -service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") +Irsa.tap_url = f"{IRSA_HOST}/TAP" # Define a query that will search the appropriate SPHEREx metadata tables # for spectral images that cover the chosen coordinates and match the @@ -120,11 +122,11 @@ service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") # selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - '{IRSA_HOST}/' || a.uri || '?center={ra.value},{dec.value}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={coord.ra.deg},{coord.dec.deg}d&size={size.value}' AS uri, p.time_bounds_lower FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid -WHERE 1 = CONTAINS(POINT('ICRS', {ra.value}, {dec.value}), p.poly) +WHERE 1 = CONTAINS(POINT('ICRS', {coord.ra.deg}, {coord.dec.deg}), p.poly) AND p.energy_bandpassname = '{bandpass}' AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower @@ -132,7 +134,7 @@ ORDER BY p.time_bounds_lower # Execute the query and return as an astropy Table. t1 = time.time() -results = service.search(query) +results = Irsa.query_tap(query) print("Time to do TAP query: {:2.2f} seconds.".format(time.time() - t1)) print("Number of images found: {}".format(len(results))) ``` @@ -148,12 +150,13 @@ This function takes in a row of the catalog that we created above and does the f - It downloads the cutout - It computes the wavelength of the center pixel of the cutout (in micro-meters) +- It keeps a copy of the cutout's original primary header and `VERSION` keyword, so that metadata from IBE is not lost when many cutouts are later combined into one MEF - It combines the image HDUs into a new HDU and adds it to the table row. Note that the values of the rows are being added in place. ```{code-cell} ipython3 -def process_cutout(row, ra, dec, cache): +def process_cutout(row, coord, cache): ''' Downloads the cutouts given in a row of the table including all SPHEREx images overlapping with a position. @@ -163,8 +166,8 @@ def process_cutout(row, ra, dec, cache): row : astropy.table row Row of a table that will be changed in place by this function. The table is created by the SQL TAP query. - ra,dec : coordinates (astropy units) - Ra and Dec coordinates (same as used for the TAP query) with attached astropy units + coord : astropy.coordinates.SkyCoord + Coordinates (same as used for the TAP query). cache : bool If set to `True`, the output of cached and the cutout processing will run faster next time. Turn this feature off by setting ``cache=False``. @@ -178,7 +181,7 @@ def process_cutout(row, ra, dec, cache): # Compute pixel coordinates corresponding to cutout position. spatial_wcs = WCS(header) - x, y = spatial_wcs.world_to_pixel(SkyCoord(ra=ra, dec=dec, unit="deg", frame="icrs")) + x, y = spatial_wcs.world_to_pixel(coord) # Compute wavelength at cutout position. spectral_wcs = WCS(header, fobj=hdulist, key="W") @@ -186,6 +189,12 @@ def process_cutout(row, ra, dec, cache): wavelength, bandwidth = spectral_wcs.pixel_to_world(x, y) row["central_wavelength"] = wavelength.to(u.micrometer).value + # IBE's primary header carries per-file metadata (e.g. VERSION) that isn't repeated + # in the extension headers. Keep a copy of it and the version string so they aren't + # silently dropped when we build the combined MEF below. + row["primary_header"] = hdulist[0].header.copy() + row["version"] = hdulist[0].header.get("VERSION", "") + # Collect the HDUs for this cutout and append the row's cutout_index to the EXTNAME. hdus = [] for hdu in hdulist[1:]: # skip the primary header @@ -199,7 +208,7 @@ It catches transient read errors and simply skips those cutouts, which is suffic For science use cases, users may instead want to implement their own retry logic or error handling strategy. ```{code-cell} ipython3 -def process_cutout_with_error_handling(row, ra, dec, cache): +def process_cutout_with_error_handling(row, coord, cache): ''' Call `process_cutout` while catching transient read errors. @@ -209,14 +218,14 @@ def process_cutout_with_error_handling(row, ra, dec, cache): row : astropy.table row Row of a table that will be changed in place by this function. The table is created by the SQL TAP query. - ra,dec : coordinates (astropy units) - Ra and Dec coordinates (same as used for the TAP query) with attached astropy units + coord : astropy.coordinates.SkyCoord + Coordinates (same as used for the TAP query). cache : bool If set to `True`, the output of cached and the cutout processing will run faster next time. Turn this feature off by setting ``cache=False``. ''' try: - process_cutout(row, ra, dec, cache=cache) + process_cutout(row, coord, cache=cache) except (TimeoutError, urllib.error.HTTPError, http.client.IncompleteRead): # Transient read errors. Skip this cutout. row["hdus"] = None @@ -254,11 +263,13 @@ To learn more about the cache please read the [astropy cache management document results_table_serial = results.to_table() results_table_serial["cutout_index"] = range(1, len(results_table_serial) + 1) results_table_serial["central_wavelength"] = np.full(len(results_table_serial), np.nan) +results_table_serial["primary_header"] = np.full(len(results_table_serial), None) +results_table_serial["version"] = np.full(len(results_table_serial), "", dtype="U32") results_table_serial["hdus"] = np.full(len(results_table_serial), None) t1 = time.time() for row in results_table_serial: - process_cutout_with_error_handling(row, ra, dec, cache=False) + process_cutout_with_error_handling(row, coord, cache=False) print("Time to create cutouts in serial mode: {:2.2f} minutes.".format((time.time() - t1) / 60)) # Drop rows that failed to download. @@ -275,16 +286,7 @@ The choice of this value depends on the number of cores but also on the number o ```{tip} A good value for the maximum number of workers is between 7 and 12 for a machine with 8 cores. -``` - -```{tip} -The astropy `fits.open()` supports a caching argument. -This can be passed through in the `process_cutout()` function. -If `cache=True` is set, the images are cached and the cutout creation is sped up next time the code is run (even if the Jupyter kernel is restarted!). -The downside is that the images are saved on the machine where this notebook is run (usually in `~/.astropy/cache/`). -If many cutouts are created, this can sum up to a large cached data volume, in which case `cache=False` is preferred. - -To learn more about the cache please read the [astropy cache management documentation](https://docs.astropy.org/en/stable/utils/data.html#cache-management). +See the caching tip in section 7.1 above -- the same `cache` argument applies here. ``` Again, before running the cutout processing we define some place holders. @@ -293,11 +295,13 @@ Again, before running the cutout processing we define some place holders. results_table_parallel = results.to_table() results_table_parallel["cutout_index"] = range(1, len(results_table_parallel) + 1) results_table_parallel["central_wavelength"] = np.full(len(results_table_parallel), np.nan) +results_table_parallel["primary_header"] = np.full(len(results_table_parallel), None) +results_table_parallel["version"] = np.full(len(results_table_parallel), "", dtype="U32") results_table_parallel["hdus"] = np.full(len(results_table_parallel), None) t1 = time.time() with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - futures = [executor.submit(process_cutout_with_error_handling, row, ra, dec, False) + futures = [executor.submit(process_cutout_with_error_handling, row, coord, False) for row in results_table_parallel] concurrent.futures.wait(futures) print("Time to create cutouts in parallel mode: {:2.2f} minutes.".format((time.time() - t1) / 60)) @@ -322,6 +326,7 @@ cols = fits.ColDefs([ fits.Column(name="cutout_index", format="J", array=results_table_parallel["cutout_index"], unit=""), fits.Column(name="observation_date", format="D", array=results_table_parallel["time_bounds_lower"], unit="d"), fits.Column(name="central_wavelength", format="D", array=results_table_parallel["central_wavelength"], unit="um"), + fits.Column(name="version", format="A32", array=results_table_parallel["version"], unit=""), fits.Column(name="access_url", format="A200", array=results_table_parallel["uri"], unit=""), ]) table_hdu = fits.BinTableHDU.from_columns(cols) @@ -331,9 +336,14 @@ table_hdu.header["EXTNAME"] = "CUTOUT_INFO" ## 9. Create the final MEF Now, we create a primary HDU and combine it with the summary table HDU and the cutout image and wavelength HDUs to a final HDU list. +A blank `fits.PrimaryHDU()` would silently drop IBE's per-file metadata (e.g. `VERSION`), so instead we base the primary header on the first cutout's -- the `version` column in `CUTOUT_INFO` above records each individual cutout's processing version in case they differ. ```{code-cell} ipython3 -primary_hdu = fits.PrimaryHDU() +primary_hdu = fits.PrimaryHDU(header=results_table_parallel["primary_header"][0]) +primary_hdu.header["HISTORY"] = ( + "Primary header copied from the first cutout in this MEF. " + "See the VERSION column in the CUTOUT_INFO table for each cutout's processing version." +) hdulist_list = [primary_hdu, table_hdu] hdulist_list.extend(hdu for fits_hdulist in results_table_parallel["hdus"] for hdu in fits_hdulist) combined_hdulist = fits.HDUList(hdulist_list) @@ -342,10 +352,11 @@ combined_hdulist = fits.HDUList(hdulist_list) ## 10. Write the final MEF Finally, we save the full HDU list to a multi-extension FITS file to disk. +We use `overwrite=False` so that re-running this cell does not silently clobber an existing output file; delete the file first (or change `output_filename` above) if you want to regenerate it. ```{code-cell} ipython3 # Write the final MEF -combined_hdulist.writeto(output_filename, overwrite=True) +combined_hdulist.writeto(output_filename, overwrite=False) ``` ## 11. Test and visualize the final result @@ -370,14 +381,18 @@ with fits.open(output_filename) as hdul: ``` Plot the images with the wavelength corresponding to their central pixel. +We read `central_wavelength` as an astropy `Quantity` (via the column's `.quantity` property, which picks up the `um` unit written into the FITS header) rather than a bare float, so the unit travels with the value instead of being assumed. ```{code-cell} ipython3 +central_wavelength = summary_table["central_wavelength"].quantity + fig = plt.figure(figsize=(15, 5)) axs = [fig.add_subplot(2, 5, ii + 1) for ii in range(10)] for ii, img in enumerate(imgs): axs[ii].imshow(imgs[ii], norm="log", origin="lower") - axs[ii].text(0.05, 0.05, r"$\lambda_{\rm center} = %2.4g \,{\rm \mu m}$" % summary_table["central_wavelength"][ii], + wl = central_wavelength[ii].to(u.micrometer) + axs[ii].text(0.05, 0.05, rf"$\lambda_{{\rm center}} = {wl.value:2.4g} \,{{\rm \mu m}}$", va="bottom", ha="left", color="white", transform=axs[ii].transAxes) plt.show() diff --git a/tutorials/spherex/spherex_psf.md b/tutorials/spherex/spherex_psf.md index 20430ede..d303a9d6 100644 --- a/tutorials/spherex/spherex_psf.md +++ b/tutorials/spherex/spherex_psf.md @@ -58,7 +58,7 @@ The following packages must be installed to run this notebook. ```{code-cell} ipython3 # Uncomment the next line to install dependencies if needed. -# !pip install astropy matplotlib numpy pyvo photutils +# !pip install astropy matplotlib numpy astroquery photutils ``` ```{code-cell} ipython3 @@ -73,11 +73,12 @@ from packaging.version import Version import astropy.units as u import matplotlib.pyplot as plt import numpy as np -import pyvo from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.table import Table +from astropy.visualization import quantity_support from astropy.wcs import WCS +from astroquery.ipac.irsa import Irsa from photutils.psf import ImagePSF @@ -85,6 +86,9 @@ from photutils.psf import ImagePSF # astropy's default timeout limit. Increase it. from astropy.utils.data import conf conf.remote_timeout = 120 + +# Let matplotlib recognize astropy Quantities and auto-label axes with their units. +quantity_support() ``` ## 4. Get SPHEREx Cutout and Obtain Position on Parent Frame @@ -104,8 +108,7 @@ To learn more about how to access SPHEREx spectral images and how to download cu ``` ```{code-cell} ipython3 -ra = 305.59875000000005 * u.degree -dec = 41.14888888888889 * u.degree +coord = SkyCoord(ra=305.59875000000005 * u.degree, dec=41.14888888888889 * u.degree, frame="icrs") size = 0.01 * u.degree ``` @@ -114,11 +117,10 @@ Once we defined the coordinates of interest and the size of the cutout, we run a ```{code-cell} ipython3 # TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and # revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +# Note: `Irsa.tap_url` must be set directly -- `astroquery.ipac.irsa.conf.tap_url` is only read +# once when the module is first imported, so changing it afterward has no effect. IRSA_HOST = "https://irsatest.ipac.caltech.edu" - -# Define the service endpoint for IRSA's Table Access Protocol (TAP) -# so that we can query SPHEREx metadata tables. -service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") +Irsa.tap_url = f"{IRSA_HOST}/TAP" # Define a query that will search the appropriate SPHEREx metadata tables # for spectral images that cover the chosen coordinates of interest. @@ -128,18 +130,18 @@ service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") # selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - '{IRSA_HOST}/' || a.uri || '?center={ra.value},{dec.value}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={coord.ra.deg},{coord.dec.deg}d&size={size.value}' AS uri, p.time_bounds_lower, p.energy_bandpassname, p.time_bounds_lower, p.time_bounds_upper, p.provenance_version FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid -WHERE 1 = CONTAINS(POINT('ICRS', {ra.value}, {dec.value}), p.poly) +WHERE 1 = CONTAINS(POINT('ICRS', {coord.ra.deg}, {coord.dec.deg}), p.poly) AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ # Execute the query and return as an astropy Table. t1 = time.time() -results = service.search(query) +results = Irsa.query_tap(query) results = results.to_table() print("Time to do TAP query: {:2.2f} seconds.".format(time.time() - t1)) print("Number of images found: {}".format(len(results))) @@ -227,7 +229,7 @@ We do this by first determining the pixel $(x,y)$ coordinates of our coordinates ```{code-cell} ipython3 wcs = WCS(cutout_header) -xpix_cutout, ypix_cutout = wcs.world_to_pixel(SkyCoord(ra=ra, dec=dec)) +xpix_cutout, ypix_cutout = wcs.world_to_pixel(coord) print(f"Pixel values of coordinates of interest on cutout image: x = {xpix_cutout}, y = {ypix_cutout}") ``` @@ -493,11 +495,13 @@ ax2.set_ylabel("$y$ PSF zones") plt.colorbar(im2 , ax=ax2, label=r"$N_{\rm eff}$ (PSF effective area)") ## As a function of wavelength -x = epsf_bintable["CWAVE"] # wavelength at the center of the PSF zone +# CWAVE has no unit metadata in the FITS table, so attach the known unit (microns) +# explicitly rather than relying on a hardcoded axis label string. +wave = epsf_bintable["CWAVE"] * u.micrometer # wavelength at the center of the PSF zone y = epsf_bintable["NEFF_MEAN"] -ax3.scatter(x , y, c=z , marker="o", s=5) +ax3.scatter(wave, y, c=z , marker="o", s=5) -ax3.set_xlabel(r"Wavelength [$\mu m$]") +ax3.set_xlabel(f"Wavelength [{wave.unit.to_string('latex_inline')}]") ax3.set_ylabel(r"$N_{\rm eff}$ (PSF effective area)") plt.show() From 61b4e2af98af5cfbf80f85642a215539ac80ee6e Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 16:19:26 -0700 Subject: [PATCH 09/22] Point spherex_intro.md at the DR1 SIA collection Replace the QR2/QR3 collection list with the DR1 wide/deep collections and query spherex_dr1. Collection names are expected, not yet confirmed: DR1 is not in SIA anywhere until the Nov 12 release. Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex_intro.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 48f6e4e8..734094c9 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -44,7 +44,7 @@ The community will also mine SPHEREx data and combine it with synergistic data s More information about SPHEREx, the data releases, and the data products is available in the [SPHEREx Archive at IRSA User Guide](https://caltech-ipac.github.io/spherex-archive-documentation). -This notebook can be used with both Quick Release 3 (QR3) and Quick Release 2 (QR2) data. +This notebook uses data from SPHEREx Data Release 1 (DR1). +++ @@ -109,15 +109,13 @@ The IRSA SIA collections can be listed using using the ``list_collections`` meth +++ The collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis) -There are currently four collections available: +The DR1 Spectral Image MEFs are available in two collections: -* `'spherex_qr3'` -- Quick Release 3 Spectral Image MEFs that are part of the SPHEREx **Wide Survey** -* `'spherex_qr3_deep'` -- Quick Release 3 Spectral Image MEFs that are part of the SPHEREx **Deep Survey** -* `'spherex_qr2'` -- Quick Release 2 Spectral Image MEFs that are part of the SPHEREx **Wide Survey** -* `'spherex_qr2_deep'` -- Quick Release 2 Spectral Image MEFs that are part of the SPHEREx **Deep Survey** +* `'spherex_dr1'` -- DR1 Spectral Image MEFs that are part of the SPHEREx **Wide Survey** +* `'spherex_dr1_deep'` -- DR1 Spectral Image MEFs that are part of the SPHEREx **Deep Survey** ```{code-cell} ipython3 -results = Irsa.query_sia(pos=(coord, search_radius), collection='spherex_qr2') +results = Irsa.query_sia(pos=(coord, search_radius), collection='spherex_dr1') ``` :::{note} From 0bf8d7cf15c89919e635df25615fc7edc5b4a99e Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 16:19:26 -0700 Subject: [PATCH 10/22] Point Source Discovery Tool at DR1 SIA and DR1 solid angle pixel maps data_release qr2 -> dr1 (also sets the SIA collection and S3 SAPM paths), and SAPM calibration cal-sapm-v2-2025-164 -> cal-sapm-v3-2026-191, the version shipped under dr1/ (verified via irsatest IBE). Co-Authored-By: Claude Opus 5.5 --- .../spherex_source_discovery_tool_demo.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index 41cbd3b3..ed54c55d 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -153,20 +153,20 @@ output_notebook(resources=INLINE) ## 5. Initialize Ephemeral Query Parameters ```{attention} -The query parameters below are valid as of **February 2026**, but will need to be updated as the mission proceeds. +The query parameters below are valid as of **SPHEREx Data Release 1 (DR1), November 2026**, but will need to be updated as the mission proceeds. Specifically, the ephemeral query parameters _will_ change for future SPHEREx data releases. ``` ```{code-cell} ipython3 # General -data_release = "qr2" +data_release = "dr1" # Spectral image queries sia_collection = f"spherex_{data_release}" # Solid angle pixel map queries dataset_type="solid_angle_pixel_map" -sapm_collection = "cal-sapm-v2-2025-164" +sapm_collection = "cal-sapm-v3-2026-191" sapm_s3 = [ f"nasa-irsa-spherex/{data_release}/{dataset_type}/{sapm_collection}/{k}/"\ f"{dataset_type}_D{k}_spx_{sapm_collection}.fits" for k in range(1, 7) @@ -194,7 +194,7 @@ results = Irsa.query_sia(pos=(coord, search_radius), collection=sia_collection) results_summary(results) ``` -Note that we have used the collection `spherex_qr2` in this case. All the available collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis). +Note that we have used the collection `spherex_dr1` in this case. All the available collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis). +++ ```{tip} From 85df1606a63fdcfffb30953e4bd9cdd6575aaf1a Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 16:39:10 -0700 Subject: [PATCH 11/22] Read QR3/DR1 FLAGS header keywords in Source Discovery Tool photometry QR3 and DR1 name flag bit n in MSKNnnnn instead of QR2's MP_ = n. build_phot_table only looked for MP_ keys, so on DR1 it recorded no flags and grab_star raised KeyError: 'TRANSIENT'. Read MSKN keys, falling back to MP_ for QR2. Also drop PHANTOM and REFERENCE from the notebook's bad_flags list; neither flag exists in DR1. Tested against DR1 on irsatest (secs. 5, 7, 8, 12): 282 cutouts, 22 flags recorded, spectrum spans 0.74-5.0 um. Co-Authored-By: Claude Opus 5.5 --- .../spherex_source_discovery_tool/aperture_photometry.py | 6 +++++- .../spherex_source_discovery_tool_demo.md | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py index 71fdc704..1f5461a8 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py @@ -393,7 +393,11 @@ def build_phot_table(results_table: Table, matched_stars: pd.DataFrame, aperture var_img = hdus[hdu_idx_map["VARIANCE"]].data flags = hdus[hdu_idx_map["FLAGS"]].data flags_header = hdus[hdu_idx_map["FLAGS"]].header - flag_dict = {key[3:]: flags_header[key] for key in flags_header if key.startswith('MP_')} + # Map flag names to bit numbers. QR3 and later name bit n in MSKNnnnn; + # QR2 used MP_ = n. + flag_dict = {flags_header[key]: int(key[4:]) for key in flags_header if key.startswith('MSKN')} + if not flag_dict: + flag_dict = {key[3:]: flags_header[key] for key in flags_header if key.startswith('MP_')} flag_dict = dict(sorted(flag_dict.items(), key=lambda x: x[1])) flag_names = list(flag_dict.keys()) flag_masks = {name: (flags & (1 << flag_dict[name])) > 0 for name in flag_names} diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index ed54c55d..fcbc59f2 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -805,8 +805,7 @@ Apply flagging to the extracted 1D-Spectrum. ```{code-cell} ipython3 # Define bad photometry flags -bad_flags = ['TRANSIENT', 'OVERFLOW', 'SUR_ERROR', 'PHANTOM', 'REFERENCE', 'NONFUNC', 'DICHROIC', - 'MISSING_DATA', 'HOT', 'COLD', 'FULLSAMPLE', 'PHANMISS', 'NONLINEAR', 'PERSIST', 'OUTLIER'] +bad_flags = ['TRANSIENT', 'OVERFLOW', 'SUR_ERROR', 'NONFUNC', 'DICHROIC', 'MISSING_DATA', 'HOT', 'COLD', 'FULLSAMPLE', 'PHANMISS', 'NONLINEAR', 'PERSIST', 'OUTLIER'] ap_rad = 3.0 # set your preferred aperture radius, from the `aperture_radii` list you specified above targ = 0 # pick index from `all_stars_band_phot` table wvl, bdw, abmags, abmag_errs, flx, var, n_bad = \ From ebd02ec2435fc3cc2ef302c1f2c9562c05d1b882 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 16:46:46 -0700 Subject: [PATCH 12/22] Build the Source Discovery Tool's D4 mask from D4 flags mask4 checked flags_data2 == 2 (the D2 image's flags) instead of flags_data4 == 2. Co-Authored-By: Claude Opus 5.5 --- .../spherex_source_discovery_tool_demo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index fcbc59f2..13222d7e 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -375,7 +375,7 @@ mask2 = np.zeros(flags_data2.shape) * np.nan mask2[(flags_data2 == 2097152) | (flags_data2 == 2) | (flags_data2 == 0)] = 1 mask4 = np.zeros(flags_data4.shape) * np.nan -mask4[(flags_data4 == 2097152) | (flags_data2 == 2) | (flags_data4 == 0)] = 1 +mask4[(flags_data4 == 2097152) | (flags_data4 == 2) | (flags_data4 == 0)] = 1 ``` Finally, we reproject the images and masks to the same pixel grid. We reproject D2 on D4. For this we use `reproject_exact()` (for flux conservation) and `reproject_interp()` (for mask) from the _reproject_ Python package. From a4ffe0da7c17b8c61cea44e25402a1bbc7394628 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 17:03:22 -0700 Subject: [PATCH 13/22] Describe the EPSF extension instead of the old PSF cube in spherex_intro.md DR1 (like QR3) replaces the 121-plane PSF image cube with an EPSF binary table: 441 rows (21 x 21 detector regions), each a 33 x 33 ePSF oversampled 5x. Update the extension list, section 12 text, and the header keywords the section 12 cell prints, and link to the ePSF tutorial. Checked against a DR1 file on irsatest. Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex_intro.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 734094c9..59453248 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -183,7 +183,7 @@ You can see that the Level 2 Spectral Image files are multi-extension FITS files * **FLAGS:** A bitmap with per-pixel status and processing flags. * **VARIANCE:** A per-pixel estimate of the variance. * **ZODI:** An model of the zodiacal dust background signal. Note that this is not subtracted from the IMAGE extension. -* **PSF:** An image cube (3D array) where each plane represents a Point Spread Function (PSF) for a cutout in over-sampled pixel space. +* **EPSF:** A table of effective Point Spread Functions (ePSFs), one for each of 441 regions of the detector, in over-sampled pixel space. * **WCS-WAVE:** Spectral WCS lookup table that maps pixel coordinates to the central wavelength and bandwidth of each pixel. We will tour each extension in the sections below. @@ -415,14 +415,14 @@ for keyword in keywords_to_print: print(f"{keyword}: {value}") ``` -## 12. Explore the fifth extension: PSF +## 12. Explore the fifth extension: EPSF -The fifth extension of the MEF contains 121 Point-spread functions (PSFs); each PSF is represented as a 101 x 101 image and all 121 are assembled together into a cube. Each of the 121 layers represents a "super-resolution" PSF estimate in a different region (defined by an 11x11 grid) of the detector. Each PSF is a two-dimensional array with size of 101 × 101 pixels. The PSFs are oversampled such that 10 PSF pixels cover the same spatial extent as one spectral image pixel (0.615 arcsec). Let's look at some specific header keywords for the PSF: +The fifth extension of the MEF contains the effective Point Spread Function (ePSF), stored as a binary table. The detector is divided into a 21 x 21 grid of regions (each about 97 x 97 spectral image pixels), and each of the 441 rows of the table holds the ePSF for one region, along with that region's position on the detector (`XCENTER`, `YCENTER`) and its central wavelength and bandwidth (`CWAVE`, `CBAND`). Each ePSF is a 33 x 33 array, oversampled by a factor of 5 relative to the spectral image pixels (header keywords `OVSMPX` and `OVSMPY`). Unlike an optical PSF, the ePSF is a model of the pixel-convolved PSF: it predicts the pixel values of a point source at any sub-pixel position. The [SPHEREx ePSF tutorial](spherex_psf.md) shows how to find and use the ePSF for a given position. Let's look at some specific header keywords for the EPSF: ```{code-cell} ipython3 -psf_header = hdulist[5].header +psf_header = hdulist["EPSF"].header -keywords_to_print = ['EXTNAME', 'NAXIS1', 'NAXIS2', 'NAXIS3', 'BUNIT'] +keywords_to_print = ['EXTNAME', 'NAXIS2', 'DETECTOR', 'OVSMPX', 'OVSMPY'] for keyword in keywords_to_print: value = psf_header.get(keyword, 'Keyword not found') From c15f929817b8c6a81f0c0e1249e033b83493ca37 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 22 Oct 2026 17:09:37 -0700 Subject: [PATCH 14/22] Update spherex_intro.md flag text and FLAGS header lookup for DR1 DR1 (like QR3) defines flags as MSKNnnnn (name) / MSKDnnnn (description) instead of MP_*, so flags_header['MP*'] printed nothing. Print a bit/name/ description table from the MSKN/MSKD keys instead. Also 14 -> 22 flags (counted in DR1 FLAGS headers), and point the Explanatory Supplement link at the renamed SPHEREx_Expsupp.pdf, dropping the QR-era table number. Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex_intro.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 59453248..90d4b66a 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -357,7 +357,7 @@ Now let's take a look at some header keywords that provide information about how spectral_image_header['L2 N_*'] ``` -There are 14 flags in total. Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The remaining flags are described in Table 8 of the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +There are 22 flags in total, defined in the FLAGS header (see section 9). Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The remaining flags are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ @@ -377,10 +377,12 @@ for keyword in keywords_to_print: print(f"{keyword}: {value}") ``` -Conveniently, the definitions of the flags are also provided in the FLAGS header. +Conveniently, the definitions of the flags are also provided in the FLAGS header. For each bit number n, the keyword `MSKNnnnn` gives the flag name and `MSKDnnnn` gives its description. ```{code-cell} ipython3 -flags_header['MP*'] +for key in flags_header['MSKN*']: + bit = key[4:] + print(f"{int(bit):2d} {flags_header[key]:<13} {flags_header['MSKD' + bit]}") ``` Tip: In the Firefly visualization, if you are looking at the first extension (IMAGE), you can open up the layers icon (top right) to enable overlaying a visualization of the flags. From 1146463915996cd1175714227b5654b17e2936cc Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 28 Oct 2026 11:00:30 -0700 Subject: [PATCH 15/22] Explain combined L2 N_* pixel-count keywords in spherex_intro.md Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex_intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 90d4b66a..44285118 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -357,7 +357,7 @@ Now let's take a look at some header keywords that provide information about how spectral_image_header['L2 N_*'] ``` -There are 22 flags in total, defined in the FLAGS header (see section 9). Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The remaining flags are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +These keywords give the number of pixels carrying each of the 22 flags defined in the FLAGS header (see section 9), plus a few combined counts: keywords ending in `_NB` count pixels that are not also flagged as bad, and those ending in `_NS` count pixels that are not also flagged SOURCE. Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The remaining flags are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ From 6615d7b7bb43af252dee798a538c87d175436d9c Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 28 Oct 2026 11:35:18 -0700 Subject: [PATCH 16/22] Use the Explanatory Supplement's bad-pixel flags in SPHEREx notebooks Source Discovery Tool: bad_flags now matches the supplement's CONTAINS_BAD_PIXEL flag list (drops FULLSAMPLE, PHANMISS and OUTLIER; adds the new DR1 artifact flags), and grab_star skips flags that an image's FLAGS header does not define. Intro: reword the L2 N_* paragraph and cite the supplement's flag table. Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex_intro.md | 2 +- .../spherex_source_discovery_tool/aperture_photometry.py | 6 ++++-- .../spherex_source_discovery_tool_demo.md | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 44285118..3d3fae69 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -357,7 +357,7 @@ Now let's take a look at some header keywords that provide information about how spectral_image_header['L2 N_*'] ``` -These keywords give the number of pixels carrying each of the 22 flags defined in the FLAGS header (see section 9), plus a few combined counts: keywords ending in `_NB` count pixels that are not also flagged as bad, and those ending in `_NS` count pixels that are not also flagged SOURCE. Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The remaining flags are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +These keywords count the pixels carrying most of the flags defined in the FLAGS header (see section 9), plus a few combined counts: keywords ending in `_NB` count pixels that are not also flagged as bad, and those ending in `_NS` count pixels that are not also flagged SOURCE. Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The flags are described in Table 16 of the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py index 1f5461a8..e3fd9e0e 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py @@ -535,7 +535,8 @@ def grab_star(obs_log_entry: list[dict], ap_rad: float, bad_flags: list[str]) -> ap_rad : `float` Aperture radius to consider. bad_flags : `list` of `str` - List of bad photometry flags to check for. + List of bad photometry flags to check for. Flags not defined in an + image's FLAGS header are skipped for that image. Returns @@ -570,7 +571,8 @@ def grab_star(obs_log_entry: list[dict], ap_rad: float, bad_flags: list[str]) -> # Sum number of bad pixels in aperture bad_pixels = 0 for flag in bad_flags: - bad_pixels += obs['dqf_sum'][flag][idx_ap] + if flag in obs['dqf_sum']: + bad_pixels += obs['dqf_sum'][flag][idx_ap] bad_flags_aperture.append(bad_pixels) # Filter out non-finite flux measurements and convert to arrays diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index 13222d7e..133ce0f2 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -802,10 +802,13 @@ if os.path.exists(phot_path): ``` Apply flagging to the extracted 1D-Spectrum. +We use the flags that set the conservative CONTAINS_BAD_PIXEL warning in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf) (flag definitions are in its Table 16). ```{code-cell} ipython3 # Define bad photometry flags -bad_flags = ['TRANSIENT', 'OVERFLOW', 'SUR_ERROR', 'NONFUNC', 'DICHROIC', 'MISSING_DATA', 'HOT', 'COLD', 'FULLSAMPLE', 'PHANMISS', 'NONLINEAR', 'PERSIST', 'OUTLIER'] +bad_flags = ['TRANSIENT', 'OVERFLOW', 'SUR_ERROR', 'NONFUNC', 'DICHROIC', 'MISSING_DATA', 'HOT', 'COLD', + 'NONLINEAR', 'CROSSTALK', 'GHOST', 'GHOST_FPA', 'GHOST_EXT', 'STREAK', 'PERSIST', + 'BLOOM', 'SNOWBALL', 'HALO', 'SATELLITE_HALO'] ap_rad = 3.0 # set your preferred aperture radius, from the `aperture_radii` list you specified above targ = 0 # pick index from `all_stars_band_phot` table wvl, bdw, abmags, abmag_errs, flx, var, n_bad = \ From 352a519e515c66b5ebde56d3799a21ded52781e1 Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 28 Oct 2026 13:42:17 -0700 Subject: [PATCH 17/22] Add a draft skill for SPHEREx data-release notebook updates Captures the DR1 update as a repeatable workflow: release-dependent spots in each notebook, settled decisions and gotchas, a release-availability check script, and a parameterized Source Discovery Tool test harness. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/spherex-data-release/SKILL.md | 118 ++++++++++++++ .../references/decisions.md | 57 +++++++ .../references/touchpoints.md | 77 +++++++++ .../scripts/check_release.sh | 55 +++++++ .../scripts/run_sdt_release_test.py | 146 ++++++++++++++++++ 5 files changed, 453 insertions(+) create mode 100644 .claude/skills/spherex-data-release/SKILL.md create mode 100644 .claude/skills/spherex-data-release/references/decisions.md create mode 100644 .claude/skills/spherex-data-release/references/touchpoints.md create mode 100755 .claude/skills/spherex-data-release/scripts/check_release.sh create mode 100644 .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py diff --git a/.claude/skills/spherex-data-release/SKILL.md b/.claude/skills/spherex-data-release/SKILL.md new file mode 100644 index 00000000..69daff10 --- /dev/null +++ b/.claude/skills/spherex-data-release/SKILL.md @@ -0,0 +1,118 @@ +--- +name: spherex-data-release +description: Update the SPHEREx tutorial notebooks in irsa-tutorials for a new SPHEREx data release (DR2, DR3, a new Quick Release, a reprocessing). Use this whenever someone mentions a new SPHEREx release, new SPHEREx collections or DOIs, a new SPHEREx Explanatory Supplement, new FLAGS bits, or asks to "update the SPHEREx notebooks", "test SPHEREx against irsatest", or prepare the SPHEREx release-day PR -- even if they only ask about one of those pieces. +--- + +# SPHEREx data-release update + +This skill repeats the work done for DR1 (issue #392, released 2026-11-12): find every place in the +SPHEREx notebooks that depends on the release, test against pre-release IRSA hosts, collect the +answers that only other people can give, and land everything in one PR on release day. + +Most of the time goes into waiting on other people, not editing code. So the main job is to keep a +written plan current, ask for missing information early, and do everything that does not depend on +those answers right away. + +## Files in this skill + +- `references/touchpoints.md` -- every release-dependent spot in each notebook, with what it looked + like for DR1. Read this first; it is the work list. +- `references/decisions.md` -- rules settled during DR1 (which release to show, which DOI to cite, + which flags count as bad, who owns which notebook) and gotchas that cost time. Read before + asking the user anything; most questions were already answered once. +- `scripts/check_release.sh ` -- what is live for release `` (e.g. `dr2`) on the + pre-release and production hosts, S3 buckets and docs. +- `scripts/run_sdt_release_test.py` -- runs the Source Discovery Tool's own notebook cells against + pre-release data, with stand-ins for the parts that are not live yet. + +## Workflow + +### 1. Start a plan file + +Create `SPHEREX__PLAN.md` at the repo root and add it (and any local test folder) to +`.git/info/exclude` so it never gets committed. Put a short "RESUME HERE" block at the top: branch, +last commit, what is blocked and on whom, next actions. Update it at the end of every work chunk. +The work spans weeks and many sessions, and this file is the only thing that carries over. + +Work on a branch named `spherex-` off `main`. Push it to the user's fork only until release; +the test-host settings must not reach `main` early. + +### 2. Find out what exists + +Run `scripts/check_release.sh ` (needs IRSA VPN for the pre-release hosts). It reports, for +TAP, SIA, IBE, S3 and the supplement PDF, whether the new release is visible. Record the results as +a "data products needed" table in the plan (product, which notebook uses it, test location, +available yes/no, date checked). Re-run it whenever something might have changed. + +### 3. Ask early for what only people can provide + +Send these questions as soon as the release date is known. Draft each as a paste-ready message +in a fenced code block. Keep each one short and specific. + +- **IRSA** (archive/backend staff): Will the new SIA collections be loaded on the pre-release host + (irsadev) before release, so the SIA notebooks can be tested? What will the collections be + named? Will `cloud_access` be filled in those rows? (On irsadev for DR1 it was empty for QR3 + rows; the Source Discovery Tool reads its S3 paths from it.) Will the data be on the staging + S3 bucket, and when does it move to the public one? +- **IRSA** (DOIs): the DOI for the new release. It is usually not available until close to release. +- **SSDC**: a copy of the new Explanatory Supplement, or confirmation that the online one is + current. Ask about any new FLAGS bits and which ones should count as bad for photometry. + +Then decide with the user what happens if the pre-release SIA load does not come through +(for DR1 the fallback was letting the tutorials go out a few days after release). Set a date to +follow up and write it in the plan. + +### 4. Make the changes that do not depend on answers + +Go through `references/touchpoints.md` notebook by notebook. For each one, make the change, then +test it against the pre-release host. In order of how early they can be done: + +1. TAP notebooks (`spherex_cutouts.md`, `spherex_psf.md`, SDT sec. 12): point at the pre-release + host through an `IRSA_HOST` variable inside a clearly marked `# TESTING ONLY` block, and add the + release filter on `a.uri`. Run them end to end with + `jupytext --to notebook --execute `. +2. Header-driven code and text: open one new-release image and compare its extensions, FLAGS + `MSKN*` keywords and `L2 N_*` counts with what the notebooks say and parse (see + touchpoints). New flags or renamed keywords break code silently -- the DR1 photometry recorded + no flags at all until the parser was fixed. +3. SIA notebooks (`spherex_intro.md`, SDT secs. 6-11): switch to the guessed collection names now + (pattern below), so only a check is left for release day. +4. Narrative: supplement link and table number, DOIs, release wording on the landing page. + +Collection names follow the pattern of the previous release: for DR1 they were guessed as +`spherex_dr1`, `spherex_dr1_deep`, `spherex_dr1_cal` from `spherex_qr3`, `spherex_qr3_deep`, +`spherex_qr3_cal`. Put a "verify the guessed names" step in the release-day checklist with the +exact lines to change if they differ. + +Commit in small, single-purpose commits and push to the fork. Ask before committing, and say in +the plan what each commit did. + +### 5. Test the Source Discovery Tool + +The SDT needs SIA results and S3 reads that do not exist before release. +`scripts/run_sdt_release_test.py` runs the notebook's own cells with stand-ins: pre-release TAP rows +instead of SIA (with a `cloud_access` value built from the URI), pre-release IBE instead of S3, and +one 2MASS star instead of the SExtractor/interactive selection. Run it from the repo root: + +```bash +.venv/bin/python .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py \ + tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md --release dr2 +``` + +Report the number of spectral points, how many are flagged, and the flux near 2.16 um against +the 2MASS K estimate. For DR1: 281 points, 40 flagged with the final `bad_flags`, about 17.9 mJy. + +### 6. Release day + +Keep a release-day checklist in the plan from the start and refine it as answers arrive. Its +final shape for DR1 is in `references/decisions.md` under "Release-day checklist". Open a draft PR +well before release so it can be reviewed, with the checklist in the PR description. Merge any +pending cleanup PRs into the release branch first so the diff shows only release work. + +As the last checklist item, update this skill with what was learned (see below). + +## Keeping this skill current + +After each release, update `references/touchpoints.md` with the new values and line numbers, add +new decisions and gotchas to `references/decisions.md`, and fix anything in this file that was +wrong. Leave out people's names: say which group to ask (IRSA, SSDC), not who. diff --git a/.claude/skills/spherex-data-release/references/decisions.md b/.claude/skills/spherex-data-release/references/decisions.md new file mode 100644 index 00000000..48f03c9e --- /dev/null +++ b/.claude/skills/spherex-data-release/references/decisions.md @@ -0,0 +1,57 @@ +# Decisions and gotchas from DR1 + +## Rules (settled; don't re-ask unless something changed) + +- **Show only the newest release.** Older collections stay searchable, but the tutorials + demonstrate only the newest one. TAP notebooks filter on `a.uri LIKE 'ibe/data/spherex//%'`. +- **DOIs:** cite only the DOI of each release the notebook actually loads (both if it loads two). +- **`bad_flags` in the Source Discovery Tool:** use the flag list that sets the supplement's + CONTAINS_BAD_PIXEL warning (supplement v2.0, p. 43). This came from the SSDC developer who wrote + the SDT. Do not use the shorter "flags for masking pixels for photometry calculations" list on + the same page. For DR1 the list was: TRANSIENT, OVERFLOW, SUR_ERROR, NONFUNC, DICHROIC, + MISSING_DATA, HOT, COLD, NONLINEAR, CROSSTALK, GHOST, GHOST_FPA, GHOST_EXT, STREAK, PERSIST, + BLOOM, SNOWBALL, HALO, SATELLITE_HALO. FULLSAMPLE is informational and was removed. +- **Supplement:** link `SPHEREx_Expsupp.pdf` (the `_QR` suffix was dropped for DR1). For flag + definitions, the online supplement is the reference -- not plans for a future release. Cite + the flag table by number (Table 16 in v2.0) and recheck the number when a new version appears. +- **Ownership:** IRSA updates the Source Discovery Tool notebook, not SSDC. +- **Test hosts** go in a `# TESTING ONLY` block with an `IRSA_HOST` variable, so release day is a + mechanical revert. Push only to a personal fork until release. +- **`cloud_access`:** assume the new SIA collection fills it. If a pre-release collection has it + empty, build the S3 path from the URI for local testing only, and don't commit that. +- **Demo mosaic cube:** not regenerated per release. + +## Gotchas + +- irsatest/irsadev are reachable only on the IRSA network or VPN. Cloud notebook servers get 403 or + cannot resolve them. +- IRSA production TAP/SIA can return 502 for a while. For non-SPHEREx lookups in tests (like + 2MASS), irsatest works too. +- `spherex.plane.datarelease` is a date. Filter by URI path or `obs_publisher_did`. +- Each release has a `_cal` SIA collection listing calibration files (ePSF, SAPM) per detector -- + the quickest way to learn calibration version names. +- IBE mirrors the S3 paths, so `s3://nasa-irsa-spherex//...` reads can be pointed at + `https:///ibe/data/spherex//...` for testing. +- Pre-release S3 data sat in a staging bucket (`nasa-irsa-spherex-dr1-staging`) that moved to the + public `nasa-irsa-spherex` on release day. +- FLAGS keywords changed between releases (`MP_*` in QR2, `MSKN*`/`MSKD*` from QR3). New flags + can appear in the supplement before they appear in image headers (SATELLITE_HALO, bit 29, was in + supplement v2.0 but not in DR1 v7.0.5 headers) -- code must tolerate missing flag names. +- Expect overlapping cleanup PRs. Merge them into the release branch before opening the release + PR, or the same changes show up twice. + +## Release-day checklist (DR1's final shape) + +1. Confirm the release is live: `scripts/check_release.sh `; verify the guessed SIA collection + names and fix intro + SDT if they differ; confirm production SIA rows have `cloud_access`. +2. Revert test hosts: delete each `# TESTING ONLY` block and `IRSA_HOST`, inline + `https://irsa.ipac.caltech.edu`, and check that + `grep -rn "irsatest\|irsadev\|IRSA_HOST" tutorials/` finds nothing. +3. Put in the new DOI (SDT acknowledgements). +4. Bump `**Updated:**` in every notebook that changed. +5. Run each notebook against production with `jupytext --to notebook --execute` (SDT needs its + conda env, `conda-spherex_sdt.yml`). +6. Re-run `techniques-and-tools/MOST_queries.md` once the release is in MOST. +7. Recheck the supplement's flag table number. +8. Mark the draft PR ready for review (it closes the release issue). +9. Update this skill with what was learned. diff --git a/.claude/skills/spherex-data-release/references/touchpoints.md b/.claude/skills/spherex-data-release/references/touchpoints.md new file mode 100644 index 00000000..0701eb9d --- /dev/null +++ b/.claude/skills/spherex-data-release/references/touchpoints.md @@ -0,0 +1,77 @@ +# Release-dependent spots in the SPHEREx notebooks + +Values shown are what DR1 used (branch `spherex-dr1`, Nov 2026). Line numbers drift; search for the +quoted text instead. Paths are relative to `tutorials/spherex/`. + +## spherex_cutouts.md (TAP + IBE cutouts) + +- `IRSA_HOST` test block (`# TESTING ONLY`): sets `IRSA_HOST` and `Irsa.tap_url`. Note + `Irsa.tap_url` must be set directly; `astroquery.ipac.irsa.conf.tap_url` is read only at import. +- TAP release filter: `AND a.uri LIKE 'ibe/data/spherex/dr1/%'`. `spherex.plane.datarelease` is a + date, not a release name, so filter on the URI path. Without a filter the query mixes releases + and `results[...][0]` picks the oldest. +- Cutout URL prefix built from `{IRSA_HOST}/` || `a.uri`. +- Supplement link (`SPHEREx_Expsupp.pdf`). +- `**Updated:**` date at the bottom. + +## spherex_psf.md (TAP + IBE cutouts + S3) + +- Same `IRSA_HOST` block, release filter and cutout URL prefix as cutouts. +- Sec. 5.1 reads the image's own `EPSF` extension (image version >= 7.0). Sec. 5.2 is a fallback + for older images that reads ePSF calibration files from S3 (`/epsf/cal-epsf-v*-*`). It must + stay guarded so it does not overwrite a new image's own ePSF (a DR1 bug). +- "primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE)" -- check against a + new image. +- Supplement link, `**Updated:**`. + +## spherex_intro.md (SIA) + +- Collection list and the `Irsa.query_sia(..., collection='spherex_dr1')` call; bullet list of + "two collections" (wide `spherex_dr1`, deep `spherex_dr1_deep`). Show only the newest release. +- "This notebook uses data from SPHEREx Data Release 1 (DR1)." +- Sec. 6 extension list and sec. 12 describe the `EPSF` bintable (DR1: 441 rows = 21x21 regions, + 33x33 ePSF, 5x oversampled via `OVSMPX`/`OVSMPY`; columns XCENTER/YCENTER/CWAVE/CBAND). +- Sec. 8d: prints `spectral_image_header['L2 N_*']` and explains it. DR1 had 26 keys: one per flag + for most flags (STREAK has none), `_NB` = not also flagged bad, `_NS` = not also flagged SOURCE, + `N_SAT_HALO` = SATELLITE_HALO. Cites the supplement's flag table (Table 16 in supplement v2.0). +- Sec. 9: loops over FLAGS header `MSKNnnnn` (name) / `MSKDnnnn` (description). QR2 used `MP_*`. + DR1 defined 22 flags. +- Supplement link, `**Updated:**`. + +## spherex_source_discovery/spherex_source_discovery_tool_demo.md (SIA + S3 + TAP) + +- Sec. 5 parameters: `data_release = "dr1"`, `sia_collection = f"spherex_{data_release}"`, + `sapm_collection = "cal-sapm-v3-2026-191"`. `data_release` also builds the S3 path + `nasa-irsa-spherex/{data_release}/solid_angle_pixel_map/{sapm_collection}/...` (folder is + `solid_angle_pixel_map/`, not `sapm/`). If only the SIA name changes, set `sia_collection` + directly. Remove any "valid as of " note that no longer holds. +- "Note that we have used the collection `spherex_dr1`" sentence after the SIA query. +- Sec. 7 `get_s3_fpath(cloud_access)`: needs `cloud_access` filled in SIA results. +- Sec. 8 masks: D2 mask from `flags_data2`, D4 mask from `flags_data4` (DR1 fixed a typo here). +- Sec. 12.1 TAP query: `IRSA_HOST` test block (uses `pyvo.dal.TAPService(f"{IRSA_HOST}/TAP")`), + release filter, cutout URL prefix. +- Sec. 12.2 `bad_flags`: the supplement's CONTAINS_BAD_PIXEL list (see decisions.md). Any listed + flag in the aperture drops the point from the plotted spectrum (`bokeh_viz.plot_spectrum`). +- Acknowledgements: DOI sentence. Cite only the DOIs of releases the notebook actually loads. +- Supplement link, `**Updated:**`. +- Package `spherex_source_discovery_tool/aperture_photometry.py`: + `build_phot_table` reads flag names from `MSKN*` (falls back to `MP_*`); `grab_star` skips + `bad_flags` entries an image does not define. Check both against new FLAGS headers. + +## spherex_mosaic.md + +- Supplement link. The static `M101_irsa.fits` demo cube was not regenerated for DR1. + +## spherex.md (landing page) + +- Wording about Quick Releases and Data Releases. + +## techniques-and-tools/MOST_queries.md + +- No release-specific code (`catalog="spherex"`). Re-run once the release is queryable in MOST to + refresh its outputs. + +## Repo files + +- `deploy_to_fornax_manifest.in` and `notebook_metadata.yml` list the SPHEREx files; update them if + files are added, renamed or removed (a stale manifest entry fails CI). diff --git a/.claude/skills/spherex-data-release/scripts/check_release.sh b/.claude/skills/spherex-data-release/scripts/check_release.sh new file mode 100755 index 00000000..f5550d41 --- /dev/null +++ b/.claude/skills/spherex-data-release/scripts/check_release.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Report what is live for a SPHEREx release on IRSA's pre-release and production hosts. +# Usage: check_release.sh e.g. check_release.sh dr2 +# The irsadev/irsatest checks need the IRSA network or VPN. +set -u +REL="${1:?usage: check_release.sh }" + +sia_collections() { + local q="SELECT DISTINCT obs_collection FROM ivoa.obscore WHERE obs_collection LIKE 'spherex%'" + curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" \ + | tail -n +2 | tr '\n' ' ' +} + +tap_rows() { + local q="SELECT TOP 1 a.uri FROM spherex.artifact a WHERE a.uri LIKE 'ibe/data/spherex/$REL/%'" + curl -s -m 120 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p +} + +cloud_access() { + local q="SELECT TOP 1 cloud_access FROM ivoa.obscore WHERE obs_collection = 'spherex_$REL'" + curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p +} + +# Subfolders under / in a bucket ("(empty)" if none, or the S3 error code). +s3_prefixes() { + local xml out + xml=$(curl -s -m 60 "https://$1.s3.us-east-1.amazonaws.com/?list-type=2&delimiter=/&prefix=$REL/") + if grep -q '' <<<"$xml"; then + grep -o '[^<]*' <<<"$xml" | sed 's/<[^>]*>//g' + return + fi + out=$(grep -o '[^<]*' <<<"$xml" | sed 's/<[^>]*>//g' | tr '\n' ' ') + echo "${out:-(empty)}" +} + +http_code() { curl -s -m 30 -o /dev/null -I -w '%{http_code}' "$1"; } + +for host in https://irsadev.ipac.caltech.edu https://irsatest.ipac.caltech.edu https://irsa.ipac.caltech.edu; do + echo "== $host" + echo " SIA collections: $(sia_collections "$host")" + uri=$(tap_rows "$host") + echo " $REL TAP row: ${uri:-(none)}" + echo " cloud_access (spherex_$REL): $(cloud_access "$host")" + # IBE has no directory listing, so check that the TAP row's file is served. + [ -n "$uri" ] && echo " IBE file for that row: HTTP $(http_code "$host/$uri")" +done + +echo "== S3" +echo " staging nasa-irsa-spherex-$REL-staging: $(s3_prefixes "nasa-irsa-spherex-$REL-staging")" +echo " public nasa-irsa-spherex: $(s3_prefixes nasa-irsa-spherex)" + +echo "== Docs" +for f in SPHEREx_Expsupp.pdf SPHEREx_Expsupp_QR.pdf; do + echo " $f: $(http_code "https://irsa.ipac.caltech.edu/data/SPHEREx/docs/$f")" +done diff --git a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py new file mode 100644 index 00000000..5b0188ef --- /dev/null +++ b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py @@ -0,0 +1,146 @@ +"""Run the Source Discovery Tool notebook's own sec. 5, 7, 8 and 12 cells against a SPHEREx +release that is not yet in SIA or public S3. + +Usage (from the repo root, IRSA network or VPN): + python run_sdt_release_test.py \ + --release dr2 [--host https://irsatest.ipac.caltech.edu] + +Stand-ins: +- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri +- s3://nasa-irsa-spherex//... reads -> the IBE mirror of the same path on --host +- secs. 10-11 (SExtractor + interactive selection) -> one 2MASS star near the field center +Secs. 9-11 (SExtractor, Firefly, bokeh) are not exercised. +""" +import argparse +import os +import sys +import json +import traceback + +import jupytext +import astropy.io.fits as fits_mod + +_ap = argparse.ArgumentParser() +_ap.add_argument("notebook") +_ap.add_argument("--release", required=True, help="e.g. dr2") +_ap.add_argument("--host", default="https://irsatest.ipac.caltech.edu") +ARGS = _ap.parse_args() +REL, HOST = ARGS.release, ARGS.host.rstrip("/") + +NB = ARGS.notebook +SDT_DIR = os.path.dirname(os.path.abspath(NB)) +sys.path.insert(0, SDT_DIR) + +S3_PREFIX = f"s3://nasa-irsa-spherex/{REL}/" +IBE_PREFIX = f"{HOST}/ibe/data/spherex/{REL}/" +_real_open = fits_mod.open + + +def open_via_ibe(name, *args, **kwargs): + if isinstance(name, str) and name.startswith(S3_PREFIX): + kwargs.pop("fsspec_kwargs", None) + name = IBE_PREFIX + name[len(S3_PREFIX):] + return _real_open(name, *args, **kwargs) + + +fits_mod.open = open_via_ibe + +cells = [c["source"] for c in jupytext.read(NB).cells if c["cell_type"] == "code"] + + +def cell(marker): + hits = [c for c in cells if marker in c] + assert len(hits) == 1, (marker, len(hits)) + return hits[0] + + +ns = {"HOST": HOST, "REL": REL} + + +def run(label, src): + print(f"\n===== {label} =====", flush=True) + try: + exec(src, ns) + return True + except Exception: + traceback.print_exc() + return False + + +run("imports", cell("import concurrent.futures")) +run("sec 5 params", cell('data_release = "')) +print("sapm_s3[1] =", ns["sapm_s3"][1]) + +# ---- sec. 6 stand-in: first D2 and D4 images of the release at the SDT position, from TAP +run("sec 6 stand-in", ''' +coord = SkyCoord(150.7817877, 3.3502204, unit='deg') +_svc = pyvo.dal.TAPService(HOST + "/TAP") +_q = """SELECT a.uri, p.energy_bandpassname, p.time_bounds_lower +FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid +WHERE 1 = CONTAINS(POINT('ICRS', 150.7817877, 3.3502204), p.poly) + AND a.uri LIKE 'ibe/data/spherex/%s/%%' +ORDER BY p.time_bounds_lower""" % REL +_t = _svc.search(_q).to_table() +print(len(_t), REL, "images at SDT position;", sorted(set(_t["energy_bandpassname"]))) +def _fake_row(det): + r = _t[_t["energy_bandpassname"] == det][0] + key = r["uri"][len("ibe/data/spherex/"):] + return {"cloud_access": json.dumps({"aws": {"bucket_name": "nasa-irsa-spherex", "key": key, + "region": "us-east-1"}})} +d2_result = _fake_row("SPHEREx-D2") +d4_result = _fake_row("SPHEREx-D4") +''') + +ok7 = run("sec 7 get_s3_fpath", cell("def get_s3_fpath")) and \ + run("sec 7 load images", cell("s3_fpath_d2 = get_s3_fpath")) +if ok7: + print("D2 path:", ns["s3_fpath_d2"]) +ok8 = ok7 and all([ + run("sec 8 SAPM", cell("s3_fpath_sapm2 = sapm_s3[1]")), + run("sec 8 units", cell('print(f"Spectral image BUNIT')), + run("sec 8 background", cell("bkg2 = sep.Background")), + run("sec 8 masks", cell("mask2 = np.zeros")), + run("sec 8 reproject", cell("img_data2_reproj, footprint = reproject_exact")), +]) +if ok8: + import numpy as np + print("D2 BUNIT now:", ns["img_hdr2"]["BUNIT"], "| reprojected finite px:", + int(np.isfinite(ns["img_data2_reproj"]).sum()), + "| good-pixel fraction D2 mask:", float(np.nanmean(ns["mask2"] == 1))) + +# ---- secs. 10-11 stand-in: one 2MASS star near the field center +run("matched_stars stand-in", ''' +import pandas as pd +# 2MASS from --host's TAP (production IRSA TAP can return 502s) +import pyvo as _pyvo +_stars = _pyvo.dal.TAPService(HOST + "/TAP").run_sync( + f"SELECT ra, dec, k_m FROM fp_psc WHERE CONTAINS(POINT('ICRS', ra, dec), " + f"CIRCLE('ICRS', {coord.ra.deg}, {coord.dec.deg}, {50/3600})) = 1").to_table() +_stars = _stars[(_stars["k_m"] > 11) & (_stars["k_m"] < 14)] +_stars.sort("k_m") +matched_stars = pd.DataFrame({"NUMBER": [65], "ra": [float(_stars["ra"][0])], "dec": [float(_stars["dec"][0])]}) +print(matched_stars, "Kmag", _stars["k_m"][0]) +''') + +ok12 = all([ + run("sec 12.1 TAP query", cell("service = pyvo.dal.TAPService")), + run("sec 12.1 process_cutout def", cell("def process_cutout")), + run("sec 12.1 results_table", cell('results_table["cutout_index"]')), + run("sec 12.1 download cutouts", cell("Process cutouts in parallel")), +]) +os.makedirs("data", exist_ok=True) +if ok12: + run("sec 12.2 build_phot_table", cell("build_phot_table(results_table")) + run("sec 12.2 load pickle", cell("with open(phot_path, 'rb')")) + if run("sec 12.2 grab_star", cell("bad_flags = [")): + print("n points:", len(ns["wvl"]), "| wl range:", ns["wvl"].min(), ns["wvl"].max()) + +if "wvl" in ns: + import numpy as np + w, f, nb = np.asarray(ns["wvl"]), np.asarray(ns["flx"]), np.asarray(ns["n_bad"]) + k = np.abs(w - 2.16) < 0.1 + print("flux near 2.16 um (mJy, median):", np.nanmedian(f[k]), "| expected for Kmag", + "~", 666.7e3 * 10 ** (-11.946 / 2.5)) + print("points with >=1 flagged pixel in aperture:", int((nb > 0).sum()), "of", len(nb)) + entry = ns["all_stars_band_phot"]["obs_log"][0][0] + print("flags recorded:", sorted(entry["dqf_sum"].keys())) From 9caed7f3ba2c19a4ae85902f57529b5a831bd3c9 Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 28 Oct 2026 17:04:49 -0700 Subject: [PATCH 18/22] Let the SPHEREx release skill test SIA against a pre-release host run_sdt_release_test.py gains --sia-host to run the notebook's own SIA cells against irsadev; decisions.md notes how to test SIA notebooks there. Co-Authored-By: Claude Opus 5.5 --- .../references/decisions.md | 5 +++++ .../scripts/run_sdt_release_test.py | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.claude/skills/spherex-data-release/references/decisions.md b/.claude/skills/spherex-data-release/references/decisions.md index 48f03c9e..abf040b6 100644 --- a/.claude/skills/spherex-data-release/references/decisions.md +++ b/.claude/skills/spherex-data-release/references/decisions.md @@ -37,6 +37,11 @@ - FLAGS keywords changed between releases (`MP_*` in QR2, `MSKN*`/`MSKD*` from QR3). New flags can appear in the supplement before they appear in image headers (SATELLITE_HALO, bit 29, was in supplement v2.0 but not in DR1 v7.0.5 headers) -- code must tolerate missing flag names. +- To test SIA notebooks against irsadev, set `Irsa.sia_url = "https://irsadev.ipac.caltech.edu/SIA"` + in a temporary copy (don't commit it). `Irsa.list_collections()` still lists production + collections, so check irsadev collections with a TAP query on `ivoa.obscore` instead. irsadev SIA + `access_url`s point at a dev IBE host. IRSA may load only part of the sky, so confirm the notebook + positions are covered. - Expect overlapping cleanup PRs. Merge them into the release branch before opening the release PR, or the same changes show up twice. diff --git a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py index 5b0188ef..3f305657 100644 --- a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py +++ b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py @@ -3,10 +3,13 @@ Usage (from the repo root, IRSA network or VPN): python run_sdt_release_test.py \ - --release dr2 [--host https://irsatest.ipac.caltech.edu] + --release dr2 [--host https://irsatest.ipac.caltech.edu] \ + [--sia-host https://irsadev.ipac.caltech.edu] Stand-ins: -- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri +- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri. + With --sia-host, sec. 6 instead runs the notebook's own SIA cells against that host (use this + once IRSA has loaded the release's SIA collections there). - s3://nasa-irsa-spherex//... reads -> the IBE mirror of the same path on --host - secs. 10-11 (SExtractor + interactive selection) -> one 2MASS star near the field center Secs. 9-11 (SExtractor, Firefly, bokeh) are not exercised. @@ -24,6 +27,7 @@ _ap.add_argument("notebook") _ap.add_argument("--release", required=True, help="e.g. dr2") _ap.add_argument("--host", default="https://irsatest.ipac.caltech.edu") +_ap.add_argument("--sia-host", help="run sec. 6 with real SIA from this host") ARGS = _ap.parse_args() REL, HOST = ARGS.release, ARGS.host.rstrip("/") @@ -71,8 +75,17 @@ def run(label, src): run("sec 5 params", cell('data_release = "')) print("sapm_s3[1] =", ns["sapm_s3"][1]) +if ARGS.sia_host: + # ---- sec. 6 for real, against the pre-release SIA host + ns["Irsa"].sia_url = ARGS.sia_host.rstrip("/") + "/SIA" + run("sec 6 coord", cell("search_radius = 50 * u.arcsec")) + run("sec 6 SIA query", cell("results = Irsa.query_sia(")) + run("sec 6 pick D2/D4", cell("d2_result = results[")) + print(len(ns["results"]), "SIA rows; D2 cloud_access:", str(ns["d2_result"]["cloud_access"])[:120]) + # ---- sec. 6 stand-in: first D2 and D4 images of the release at the SDT position, from TAP -run("sec 6 stand-in", ''' +else: + run("sec 6 stand-in", ''' coord = SkyCoord(150.7817877, 3.3502204, unit='deg') _svc = pyvo.dal.TAPService(HOST + "/TAP") _q = """SELECT a.uri, p.energy_bandpassname, p.time_bounds_lower From 50018d28b7c3f184e15b0d1c27c7f8a3dfda3048 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 29 Oct 2026 12:21:00 -0700 Subject: [PATCH 19/22] Move the SPHEREx release skill to its own branch The skill gets its own PR (branch spherex-release-skill) so the DR1 tutorial PR covers only notebook changes. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/spherex-data-release/SKILL.md | 118 ------------- .../references/decisions.md | 62 ------- .../references/touchpoints.md | 77 --------- .../scripts/check_release.sh | 55 ------ .../scripts/run_sdt_release_test.py | 159 ------------------ 5 files changed, 471 deletions(-) delete mode 100644 .claude/skills/spherex-data-release/SKILL.md delete mode 100644 .claude/skills/spherex-data-release/references/decisions.md delete mode 100644 .claude/skills/spherex-data-release/references/touchpoints.md delete mode 100755 .claude/skills/spherex-data-release/scripts/check_release.sh delete mode 100644 .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py diff --git a/.claude/skills/spherex-data-release/SKILL.md b/.claude/skills/spherex-data-release/SKILL.md deleted file mode 100644 index 69daff10..00000000 --- a/.claude/skills/spherex-data-release/SKILL.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -name: spherex-data-release -description: Update the SPHEREx tutorial notebooks in irsa-tutorials for a new SPHEREx data release (DR2, DR3, a new Quick Release, a reprocessing). Use this whenever someone mentions a new SPHEREx release, new SPHEREx collections or DOIs, a new SPHEREx Explanatory Supplement, new FLAGS bits, or asks to "update the SPHEREx notebooks", "test SPHEREx against irsatest", or prepare the SPHEREx release-day PR -- even if they only ask about one of those pieces. ---- - -# SPHEREx data-release update - -This skill repeats the work done for DR1 (issue #392, released 2026-11-12): find every place in the -SPHEREx notebooks that depends on the release, test against pre-release IRSA hosts, collect the -answers that only other people can give, and land everything in one PR on release day. - -Most of the time goes into waiting on other people, not editing code. So the main job is to keep a -written plan current, ask for missing information early, and do everything that does not depend on -those answers right away. - -## Files in this skill - -- `references/touchpoints.md` -- every release-dependent spot in each notebook, with what it looked - like for DR1. Read this first; it is the work list. -- `references/decisions.md` -- rules settled during DR1 (which release to show, which DOI to cite, - which flags count as bad, who owns which notebook) and gotchas that cost time. Read before - asking the user anything; most questions were already answered once. -- `scripts/check_release.sh ` -- what is live for release `` (e.g. `dr2`) on the - pre-release and production hosts, S3 buckets and docs. -- `scripts/run_sdt_release_test.py` -- runs the Source Discovery Tool's own notebook cells against - pre-release data, with stand-ins for the parts that are not live yet. - -## Workflow - -### 1. Start a plan file - -Create `SPHEREX__PLAN.md` at the repo root and add it (and any local test folder) to -`.git/info/exclude` so it never gets committed. Put a short "RESUME HERE" block at the top: branch, -last commit, what is blocked and on whom, next actions. Update it at the end of every work chunk. -The work spans weeks and many sessions, and this file is the only thing that carries over. - -Work on a branch named `spherex-` off `main`. Push it to the user's fork only until release; -the test-host settings must not reach `main` early. - -### 2. Find out what exists - -Run `scripts/check_release.sh ` (needs IRSA VPN for the pre-release hosts). It reports, for -TAP, SIA, IBE, S3 and the supplement PDF, whether the new release is visible. Record the results as -a "data products needed" table in the plan (product, which notebook uses it, test location, -available yes/no, date checked). Re-run it whenever something might have changed. - -### 3. Ask early for what only people can provide - -Send these questions as soon as the release date is known. Draft each as a paste-ready message -in a fenced code block. Keep each one short and specific. - -- **IRSA** (archive/backend staff): Will the new SIA collections be loaded on the pre-release host - (irsadev) before release, so the SIA notebooks can be tested? What will the collections be - named? Will `cloud_access` be filled in those rows? (On irsadev for DR1 it was empty for QR3 - rows; the Source Discovery Tool reads its S3 paths from it.) Will the data be on the staging - S3 bucket, and when does it move to the public one? -- **IRSA** (DOIs): the DOI for the new release. It is usually not available until close to release. -- **SSDC**: a copy of the new Explanatory Supplement, or confirmation that the online one is - current. Ask about any new FLAGS bits and which ones should count as bad for photometry. - -Then decide with the user what happens if the pre-release SIA load does not come through -(for DR1 the fallback was letting the tutorials go out a few days after release). Set a date to -follow up and write it in the plan. - -### 4. Make the changes that do not depend on answers - -Go through `references/touchpoints.md` notebook by notebook. For each one, make the change, then -test it against the pre-release host. In order of how early they can be done: - -1. TAP notebooks (`spherex_cutouts.md`, `spherex_psf.md`, SDT sec. 12): point at the pre-release - host through an `IRSA_HOST` variable inside a clearly marked `# TESTING ONLY` block, and add the - release filter on `a.uri`. Run them end to end with - `jupytext --to notebook --execute `. -2. Header-driven code and text: open one new-release image and compare its extensions, FLAGS - `MSKN*` keywords and `L2 N_*` counts with what the notebooks say and parse (see - touchpoints). New flags or renamed keywords break code silently -- the DR1 photometry recorded - no flags at all until the parser was fixed. -3. SIA notebooks (`spherex_intro.md`, SDT secs. 6-11): switch to the guessed collection names now - (pattern below), so only a check is left for release day. -4. Narrative: supplement link and table number, DOIs, release wording on the landing page. - -Collection names follow the pattern of the previous release: for DR1 they were guessed as -`spherex_dr1`, `spherex_dr1_deep`, `spherex_dr1_cal` from `spherex_qr3`, `spherex_qr3_deep`, -`spherex_qr3_cal`. Put a "verify the guessed names" step in the release-day checklist with the -exact lines to change if they differ. - -Commit in small, single-purpose commits and push to the fork. Ask before committing, and say in -the plan what each commit did. - -### 5. Test the Source Discovery Tool - -The SDT needs SIA results and S3 reads that do not exist before release. -`scripts/run_sdt_release_test.py` runs the notebook's own cells with stand-ins: pre-release TAP rows -instead of SIA (with a `cloud_access` value built from the URI), pre-release IBE instead of S3, and -one 2MASS star instead of the SExtractor/interactive selection. Run it from the repo root: - -```bash -.venv/bin/python .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py \ - tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md --release dr2 -``` - -Report the number of spectral points, how many are flagged, and the flux near 2.16 um against -the 2MASS K estimate. For DR1: 281 points, 40 flagged with the final `bad_flags`, about 17.9 mJy. - -### 6. Release day - -Keep a release-day checklist in the plan from the start and refine it as answers arrive. Its -final shape for DR1 is in `references/decisions.md` under "Release-day checklist". Open a draft PR -well before release so it can be reviewed, with the checklist in the PR description. Merge any -pending cleanup PRs into the release branch first so the diff shows only release work. - -As the last checklist item, update this skill with what was learned (see below). - -## Keeping this skill current - -After each release, update `references/touchpoints.md` with the new values and line numbers, add -new decisions and gotchas to `references/decisions.md`, and fix anything in this file that was -wrong. Leave out people's names: say which group to ask (IRSA, SSDC), not who. diff --git a/.claude/skills/spherex-data-release/references/decisions.md b/.claude/skills/spherex-data-release/references/decisions.md deleted file mode 100644 index abf040b6..00000000 --- a/.claude/skills/spherex-data-release/references/decisions.md +++ /dev/null @@ -1,62 +0,0 @@ -# Decisions and gotchas from DR1 - -## Rules (settled; don't re-ask unless something changed) - -- **Show only the newest release.** Older collections stay searchable, but the tutorials - demonstrate only the newest one. TAP notebooks filter on `a.uri LIKE 'ibe/data/spherex//%'`. -- **DOIs:** cite only the DOI of each release the notebook actually loads (both if it loads two). -- **`bad_flags` in the Source Discovery Tool:** use the flag list that sets the supplement's - CONTAINS_BAD_PIXEL warning (supplement v2.0, p. 43). This came from the SSDC developer who wrote - the SDT. Do not use the shorter "flags for masking pixels for photometry calculations" list on - the same page. For DR1 the list was: TRANSIENT, OVERFLOW, SUR_ERROR, NONFUNC, DICHROIC, - MISSING_DATA, HOT, COLD, NONLINEAR, CROSSTALK, GHOST, GHOST_FPA, GHOST_EXT, STREAK, PERSIST, - BLOOM, SNOWBALL, HALO, SATELLITE_HALO. FULLSAMPLE is informational and was removed. -- **Supplement:** link `SPHEREx_Expsupp.pdf` (the `_QR` suffix was dropped for DR1). For flag - definitions, the online supplement is the reference -- not plans for a future release. Cite - the flag table by number (Table 16 in v2.0) and recheck the number when a new version appears. -- **Ownership:** IRSA updates the Source Discovery Tool notebook, not SSDC. -- **Test hosts** go in a `# TESTING ONLY` block with an `IRSA_HOST` variable, so release day is a - mechanical revert. Push only to a personal fork until release. -- **`cloud_access`:** assume the new SIA collection fills it. If a pre-release collection has it - empty, build the S3 path from the URI for local testing only, and don't commit that. -- **Demo mosaic cube:** not regenerated per release. - -## Gotchas - -- irsatest/irsadev are reachable only on the IRSA network or VPN. Cloud notebook servers get 403 or - cannot resolve them. -- IRSA production TAP/SIA can return 502 for a while. For non-SPHEREx lookups in tests (like - 2MASS), irsatest works too. -- `spherex.plane.datarelease` is a date. Filter by URI path or `obs_publisher_did`. -- Each release has a `_cal` SIA collection listing calibration files (ePSF, SAPM) per detector -- - the quickest way to learn calibration version names. -- IBE mirrors the S3 paths, so `s3://nasa-irsa-spherex//...` reads can be pointed at - `https:///ibe/data/spherex//...` for testing. -- Pre-release S3 data sat in a staging bucket (`nasa-irsa-spherex-dr1-staging`) that moved to the - public `nasa-irsa-spherex` on release day. -- FLAGS keywords changed between releases (`MP_*` in QR2, `MSKN*`/`MSKD*` from QR3). New flags - can appear in the supplement before they appear in image headers (SATELLITE_HALO, bit 29, was in - supplement v2.0 but not in DR1 v7.0.5 headers) -- code must tolerate missing flag names. -- To test SIA notebooks against irsadev, set `Irsa.sia_url = "https://irsadev.ipac.caltech.edu/SIA"` - in a temporary copy (don't commit it). `Irsa.list_collections()` still lists production - collections, so check irsadev collections with a TAP query on `ivoa.obscore` instead. irsadev SIA - `access_url`s point at a dev IBE host. IRSA may load only part of the sky, so confirm the notebook - positions are covered. -- Expect overlapping cleanup PRs. Merge them into the release branch before opening the release - PR, or the same changes show up twice. - -## Release-day checklist (DR1's final shape) - -1. Confirm the release is live: `scripts/check_release.sh `; verify the guessed SIA collection - names and fix intro + SDT if they differ; confirm production SIA rows have `cloud_access`. -2. Revert test hosts: delete each `# TESTING ONLY` block and `IRSA_HOST`, inline - `https://irsa.ipac.caltech.edu`, and check that - `grep -rn "irsatest\|irsadev\|IRSA_HOST" tutorials/` finds nothing. -3. Put in the new DOI (SDT acknowledgements). -4. Bump `**Updated:**` in every notebook that changed. -5. Run each notebook against production with `jupytext --to notebook --execute` (SDT needs its - conda env, `conda-spherex_sdt.yml`). -6. Re-run `techniques-and-tools/MOST_queries.md` once the release is in MOST. -7. Recheck the supplement's flag table number. -8. Mark the draft PR ready for review (it closes the release issue). -9. Update this skill with what was learned. diff --git a/.claude/skills/spherex-data-release/references/touchpoints.md b/.claude/skills/spherex-data-release/references/touchpoints.md deleted file mode 100644 index 0701eb9d..00000000 --- a/.claude/skills/spherex-data-release/references/touchpoints.md +++ /dev/null @@ -1,77 +0,0 @@ -# Release-dependent spots in the SPHEREx notebooks - -Values shown are what DR1 used (branch `spherex-dr1`, Nov 2026). Line numbers drift; search for the -quoted text instead. Paths are relative to `tutorials/spherex/`. - -## spherex_cutouts.md (TAP + IBE cutouts) - -- `IRSA_HOST` test block (`# TESTING ONLY`): sets `IRSA_HOST` and `Irsa.tap_url`. Note - `Irsa.tap_url` must be set directly; `astroquery.ipac.irsa.conf.tap_url` is read only at import. -- TAP release filter: `AND a.uri LIKE 'ibe/data/spherex/dr1/%'`. `spherex.plane.datarelease` is a - date, not a release name, so filter on the URI path. Without a filter the query mixes releases - and `results[...][0]` picks the oldest. -- Cutout URL prefix built from `{IRSA_HOST}/` || `a.uri`. -- Supplement link (`SPHEREx_Expsupp.pdf`). -- `**Updated:**` date at the bottom. - -## spherex_psf.md (TAP + IBE cutouts + S3) - -- Same `IRSA_HOST` block, release filter and cutout URL prefix as cutouts. -- Sec. 5.1 reads the image's own `EPSF` extension (image version >= 7.0). Sec. 5.2 is a fallback - for older images that reads ePSF calibration files from S3 (`/epsf/cal-epsf-v*-*`). It must - stay guarded so it does not overwrite a new image's own ePSF (a DR1 bug). -- "primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE)" -- check against a - new image. -- Supplement link, `**Updated:**`. - -## spherex_intro.md (SIA) - -- Collection list and the `Irsa.query_sia(..., collection='spherex_dr1')` call; bullet list of - "two collections" (wide `spherex_dr1`, deep `spherex_dr1_deep`). Show only the newest release. -- "This notebook uses data from SPHEREx Data Release 1 (DR1)." -- Sec. 6 extension list and sec. 12 describe the `EPSF` bintable (DR1: 441 rows = 21x21 regions, - 33x33 ePSF, 5x oversampled via `OVSMPX`/`OVSMPY`; columns XCENTER/YCENTER/CWAVE/CBAND). -- Sec. 8d: prints `spectral_image_header['L2 N_*']` and explains it. DR1 had 26 keys: one per flag - for most flags (STREAK has none), `_NB` = not also flagged bad, `_NS` = not also flagged SOURCE, - `N_SAT_HALO` = SATELLITE_HALO. Cites the supplement's flag table (Table 16 in supplement v2.0). -- Sec. 9: loops over FLAGS header `MSKNnnnn` (name) / `MSKDnnnn` (description). QR2 used `MP_*`. - DR1 defined 22 flags. -- Supplement link, `**Updated:**`. - -## spherex_source_discovery/spherex_source_discovery_tool_demo.md (SIA + S3 + TAP) - -- Sec. 5 parameters: `data_release = "dr1"`, `sia_collection = f"spherex_{data_release}"`, - `sapm_collection = "cal-sapm-v3-2026-191"`. `data_release` also builds the S3 path - `nasa-irsa-spherex/{data_release}/solid_angle_pixel_map/{sapm_collection}/...` (folder is - `solid_angle_pixel_map/`, not `sapm/`). If only the SIA name changes, set `sia_collection` - directly. Remove any "valid as of " note that no longer holds. -- "Note that we have used the collection `spherex_dr1`" sentence after the SIA query. -- Sec. 7 `get_s3_fpath(cloud_access)`: needs `cloud_access` filled in SIA results. -- Sec. 8 masks: D2 mask from `flags_data2`, D4 mask from `flags_data4` (DR1 fixed a typo here). -- Sec. 12.1 TAP query: `IRSA_HOST` test block (uses `pyvo.dal.TAPService(f"{IRSA_HOST}/TAP")`), - release filter, cutout URL prefix. -- Sec. 12.2 `bad_flags`: the supplement's CONTAINS_BAD_PIXEL list (see decisions.md). Any listed - flag in the aperture drops the point from the plotted spectrum (`bokeh_viz.plot_spectrum`). -- Acknowledgements: DOI sentence. Cite only the DOIs of releases the notebook actually loads. -- Supplement link, `**Updated:**`. -- Package `spherex_source_discovery_tool/aperture_photometry.py`: - `build_phot_table` reads flag names from `MSKN*` (falls back to `MP_*`); `grab_star` skips - `bad_flags` entries an image does not define. Check both against new FLAGS headers. - -## spherex_mosaic.md - -- Supplement link. The static `M101_irsa.fits` demo cube was not regenerated for DR1. - -## spherex.md (landing page) - -- Wording about Quick Releases and Data Releases. - -## techniques-and-tools/MOST_queries.md - -- No release-specific code (`catalog="spherex"`). Re-run once the release is queryable in MOST to - refresh its outputs. - -## Repo files - -- `deploy_to_fornax_manifest.in` and `notebook_metadata.yml` list the SPHEREx files; update them if - files are added, renamed or removed (a stale manifest entry fails CI). diff --git a/.claude/skills/spherex-data-release/scripts/check_release.sh b/.claude/skills/spherex-data-release/scripts/check_release.sh deleted file mode 100755 index f5550d41..00000000 --- a/.claude/skills/spherex-data-release/scripts/check_release.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Report what is live for a SPHEREx release on IRSA's pre-release and production hosts. -# Usage: check_release.sh e.g. check_release.sh dr2 -# The irsadev/irsatest checks need the IRSA network or VPN. -set -u -REL="${1:?usage: check_release.sh }" - -sia_collections() { - local q="SELECT DISTINCT obs_collection FROM ivoa.obscore WHERE obs_collection LIKE 'spherex%'" - curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" \ - | tail -n +2 | tr '\n' ' ' -} - -tap_rows() { - local q="SELECT TOP 1 a.uri FROM spherex.artifact a WHERE a.uri LIKE 'ibe/data/spherex/$REL/%'" - curl -s -m 120 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p -} - -cloud_access() { - local q="SELECT TOP 1 cloud_access FROM ivoa.obscore WHERE obs_collection = 'spherex_$REL'" - curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p -} - -# Subfolders under / in a bucket ("(empty)" if none, or the S3 error code). -s3_prefixes() { - local xml out - xml=$(curl -s -m 60 "https://$1.s3.us-east-1.amazonaws.com/?list-type=2&delimiter=/&prefix=$REL/") - if grep -q '' <<<"$xml"; then - grep -o '[^<]*' <<<"$xml" | sed 's/<[^>]*>//g' - return - fi - out=$(grep -o '[^<]*' <<<"$xml" | sed 's/<[^>]*>//g' | tr '\n' ' ') - echo "${out:-(empty)}" -} - -http_code() { curl -s -m 30 -o /dev/null -I -w '%{http_code}' "$1"; } - -for host in https://irsadev.ipac.caltech.edu https://irsatest.ipac.caltech.edu https://irsa.ipac.caltech.edu; do - echo "== $host" - echo " SIA collections: $(sia_collections "$host")" - uri=$(tap_rows "$host") - echo " $REL TAP row: ${uri:-(none)}" - echo " cloud_access (spherex_$REL): $(cloud_access "$host")" - # IBE has no directory listing, so check that the TAP row's file is served. - [ -n "$uri" ] && echo " IBE file for that row: HTTP $(http_code "$host/$uri")" -done - -echo "== S3" -echo " staging nasa-irsa-spherex-$REL-staging: $(s3_prefixes "nasa-irsa-spherex-$REL-staging")" -echo " public nasa-irsa-spherex: $(s3_prefixes nasa-irsa-spherex)" - -echo "== Docs" -for f in SPHEREx_Expsupp.pdf SPHEREx_Expsupp_QR.pdf; do - echo " $f: $(http_code "https://irsa.ipac.caltech.edu/data/SPHEREx/docs/$f")" -done diff --git a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py deleted file mode 100644 index 3f305657..00000000 --- a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Run the Source Discovery Tool notebook's own sec. 5, 7, 8 and 12 cells against a SPHEREx -release that is not yet in SIA or public S3. - -Usage (from the repo root, IRSA network or VPN): - python run_sdt_release_test.py \ - --release dr2 [--host https://irsatest.ipac.caltech.edu] \ - [--sia-host https://irsadev.ipac.caltech.edu] - -Stand-ins: -- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri. - With --sia-host, sec. 6 instead runs the notebook's own SIA cells against that host (use this - once IRSA has loaded the release's SIA collections there). -- s3://nasa-irsa-spherex//... reads -> the IBE mirror of the same path on --host -- secs. 10-11 (SExtractor + interactive selection) -> one 2MASS star near the field center -Secs. 9-11 (SExtractor, Firefly, bokeh) are not exercised. -""" -import argparse -import os -import sys -import json -import traceback - -import jupytext -import astropy.io.fits as fits_mod - -_ap = argparse.ArgumentParser() -_ap.add_argument("notebook") -_ap.add_argument("--release", required=True, help="e.g. dr2") -_ap.add_argument("--host", default="https://irsatest.ipac.caltech.edu") -_ap.add_argument("--sia-host", help="run sec. 6 with real SIA from this host") -ARGS = _ap.parse_args() -REL, HOST = ARGS.release, ARGS.host.rstrip("/") - -NB = ARGS.notebook -SDT_DIR = os.path.dirname(os.path.abspath(NB)) -sys.path.insert(0, SDT_DIR) - -S3_PREFIX = f"s3://nasa-irsa-spherex/{REL}/" -IBE_PREFIX = f"{HOST}/ibe/data/spherex/{REL}/" -_real_open = fits_mod.open - - -def open_via_ibe(name, *args, **kwargs): - if isinstance(name, str) and name.startswith(S3_PREFIX): - kwargs.pop("fsspec_kwargs", None) - name = IBE_PREFIX + name[len(S3_PREFIX):] - return _real_open(name, *args, **kwargs) - - -fits_mod.open = open_via_ibe - -cells = [c["source"] for c in jupytext.read(NB).cells if c["cell_type"] == "code"] - - -def cell(marker): - hits = [c for c in cells if marker in c] - assert len(hits) == 1, (marker, len(hits)) - return hits[0] - - -ns = {"HOST": HOST, "REL": REL} - - -def run(label, src): - print(f"\n===== {label} =====", flush=True) - try: - exec(src, ns) - return True - except Exception: - traceback.print_exc() - return False - - -run("imports", cell("import concurrent.futures")) -run("sec 5 params", cell('data_release = "')) -print("sapm_s3[1] =", ns["sapm_s3"][1]) - -if ARGS.sia_host: - # ---- sec. 6 for real, against the pre-release SIA host - ns["Irsa"].sia_url = ARGS.sia_host.rstrip("/") + "/SIA" - run("sec 6 coord", cell("search_radius = 50 * u.arcsec")) - run("sec 6 SIA query", cell("results = Irsa.query_sia(")) - run("sec 6 pick D2/D4", cell("d2_result = results[")) - print(len(ns["results"]), "SIA rows; D2 cloud_access:", str(ns["d2_result"]["cloud_access"])[:120]) - -# ---- sec. 6 stand-in: first D2 and D4 images of the release at the SDT position, from TAP -else: - run("sec 6 stand-in", ''' -coord = SkyCoord(150.7817877, 3.3502204, unit='deg') -_svc = pyvo.dal.TAPService(HOST + "/TAP") -_q = """SELECT a.uri, p.energy_bandpassname, p.time_bounds_lower -FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid -WHERE 1 = CONTAINS(POINT('ICRS', 150.7817877, 3.3502204), p.poly) - AND a.uri LIKE 'ibe/data/spherex/%s/%%' -ORDER BY p.time_bounds_lower""" % REL -_t = _svc.search(_q).to_table() -print(len(_t), REL, "images at SDT position;", sorted(set(_t["energy_bandpassname"]))) -def _fake_row(det): - r = _t[_t["energy_bandpassname"] == det][0] - key = r["uri"][len("ibe/data/spherex/"):] - return {"cloud_access": json.dumps({"aws": {"bucket_name": "nasa-irsa-spherex", "key": key, - "region": "us-east-1"}})} -d2_result = _fake_row("SPHEREx-D2") -d4_result = _fake_row("SPHEREx-D4") -''') - -ok7 = run("sec 7 get_s3_fpath", cell("def get_s3_fpath")) and \ - run("sec 7 load images", cell("s3_fpath_d2 = get_s3_fpath")) -if ok7: - print("D2 path:", ns["s3_fpath_d2"]) -ok8 = ok7 and all([ - run("sec 8 SAPM", cell("s3_fpath_sapm2 = sapm_s3[1]")), - run("sec 8 units", cell('print(f"Spectral image BUNIT')), - run("sec 8 background", cell("bkg2 = sep.Background")), - run("sec 8 masks", cell("mask2 = np.zeros")), - run("sec 8 reproject", cell("img_data2_reproj, footprint = reproject_exact")), -]) -if ok8: - import numpy as np - print("D2 BUNIT now:", ns["img_hdr2"]["BUNIT"], "| reprojected finite px:", - int(np.isfinite(ns["img_data2_reproj"]).sum()), - "| good-pixel fraction D2 mask:", float(np.nanmean(ns["mask2"] == 1))) - -# ---- secs. 10-11 stand-in: one 2MASS star near the field center -run("matched_stars stand-in", ''' -import pandas as pd -# 2MASS from --host's TAP (production IRSA TAP can return 502s) -import pyvo as _pyvo -_stars = _pyvo.dal.TAPService(HOST + "/TAP").run_sync( - f"SELECT ra, dec, k_m FROM fp_psc WHERE CONTAINS(POINT('ICRS', ra, dec), " - f"CIRCLE('ICRS', {coord.ra.deg}, {coord.dec.deg}, {50/3600})) = 1").to_table() -_stars = _stars[(_stars["k_m"] > 11) & (_stars["k_m"] < 14)] -_stars.sort("k_m") -matched_stars = pd.DataFrame({"NUMBER": [65], "ra": [float(_stars["ra"][0])], "dec": [float(_stars["dec"][0])]}) -print(matched_stars, "Kmag", _stars["k_m"][0]) -''') - -ok12 = all([ - run("sec 12.1 TAP query", cell("service = pyvo.dal.TAPService")), - run("sec 12.1 process_cutout def", cell("def process_cutout")), - run("sec 12.1 results_table", cell('results_table["cutout_index"]')), - run("sec 12.1 download cutouts", cell("Process cutouts in parallel")), -]) -os.makedirs("data", exist_ok=True) -if ok12: - run("sec 12.2 build_phot_table", cell("build_phot_table(results_table")) - run("sec 12.2 load pickle", cell("with open(phot_path, 'rb')")) - if run("sec 12.2 grab_star", cell("bad_flags = [")): - print("n points:", len(ns["wvl"]), "| wl range:", ns["wvl"].min(), ns["wvl"].max()) - -if "wvl" in ns: - import numpy as np - w, f, nb = np.asarray(ns["wvl"]), np.asarray(ns["flx"]), np.asarray(ns["n_bad"]) - k = np.abs(w - 2.16) < 0.1 - print("flux near 2.16 um (mJy, median):", np.nanmedian(f[k]), "| expected for Kmag", - "~", 666.7e3 * 10 ** (-11.946 / 2.5)) - print("points with >=1 flagged pixel in aperture:", int((nb > 0).sum()), "of", len(nb)) - entry = ns["all_stars_band_phot"]["obs_log"][0][0] - print("flags recorded:", sorted(entry["dqf_sum"].keys())) From 245086e8addbfaf920ce19c162dbdb91ab5dd2df Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 29 Oct 2026 12:45:19 -0700 Subject: [PATCH 20/22] Select the Source Discovery Tool's D4 image by detector name The D4 image was taken from groups[3], which is D4 only when all six detectors have images at the search position. At the demo position, DR1 returns only D1, D2, D4 and D5, so groups[3] silently picked D5. Co-Authored-By: Claude Opus 5.5 --- .../spherex_source_discovery_tool_demo.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index 3632b9a3..6152120f 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -270,7 +270,10 @@ d2_result = results[results['energy_bandpassname'] == 'SPHEREx-D2'][0] # first # Using `astropy.table.TableGroups` results_by_detector = results.group_by('energy_bandpassname') print(results_by_detector.groups.keys) -d4_result = results_by_detector.groups[3][0] # first D4 image +# Select the group by name: not every detector has images at every position, +# so the D4 group is not always at the same index +d4_group = results_by_detector.groups.keys['energy_bandpassname'] == 'SPHEREx-D4' +d4_result = results_by_detector.groups[d4_group][0] # first D4 image ``` ## 7. Load the Data From 1a8231a9003369347607609ec9147ce8bf3e4d97 Mon Sep 17 00:00:00 2001 From: Jessica Krick Date: Wed, 4 Nov 2026 18:46:58 -0500 Subject: [PATCH 21/22] Say QR3 and DR1 supersede QR2 in SPHEREx notebooks Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex.md | 1 + tutorials/spherex/spherex_cutouts.md | 2 ++ tutorials/spherex/spherex_intro.md | 1 + tutorials/spherex/spherex_psf.md | 4 +++- .../spherex_source_discovery_tool_demo.md | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tutorials/spherex/spherex.md b/tutorials/spherex/spherex.md index 774070ca..ad4db281 100644 --- a/tutorials/spherex/spherex.md +++ b/tutorials/spherex/spherex.md @@ -5,6 +5,7 @@ SPHEREx observes the sky from roughly 0.75–5.0 µm using a single instrument t Its science goals span cosmology, galaxy evolution, and the interstellar medium, enabling measurements of large-scale structure, the cosmic history of star formation, and the distribution of key molecules and ices in the Milky Way and nearby galaxies. SPHEREx data releases include weekly [Quick Release spectral image products](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-products/) as well as periodic Data Releases (e.g., DR1) (multi-extension FITS files containing calibrated near-infrared surface brightness, variance, flags, modeled backgrounds, PSFs, and wavelength WCS) along with ancillary calibration and metadata files such as gain matrices, dark current maps, solid angle pixel maps, and detailed spectral WCS products for each detector. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. ```{notebook-gallery} notebook_metadata.yml tutorials/spherex/spherex_intro.md diff --git a/tutorials/spherex/spherex_cutouts.md b/tutorials/spherex/spherex_cutouts.md index 561c149d..b9d90549 100644 --- a/tutorials/spherex/spherex_cutouts.md +++ b/tutorials/spherex/spherex_cutouts.md @@ -104,6 +104,8 @@ output_filename = 'spherex_cutouts_mef.fits' Here we show how to use `astroquery`'s IRSA TAP interface to retrieve all images that overlap with the position defined above. This query will retrieve a table of URLs that link to the MEF cutouts. Each row in the table corresponds to a single cutout and includes the data access URL and an observation timestamp. +The SPHEREx metadata tables include images from every release. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2; this query keeps only DR1 images. The results are sorted from oldest to newest. ```{code-cell} ipython3 diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 5d3ef97d..b5714181 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -45,6 +45,7 @@ The community will also mine SPHEREx data and combine it with synergistic data s More information about SPHEREx, the data releases, and the data products is available in the [SPHEREx Archive at IRSA User Guide](https://caltech-ipac.github.io/spherex-archive-documentation). This notebook uses data from SPHEREx Data Release 1 (DR1). +QR3 and DR1 supersede the earlier Quick Release 2 (QR2), and SPHEREx recommends using QR3 and DR1 over QR2. +++ diff --git a/tutorials/spherex/spherex_psf.md b/tutorials/spherex/spherex_psf.md index d8f8073f..c9d487aa 100644 --- a/tutorials/spherex/spherex_psf.md +++ b/tutorials/spherex/spherex_psf.md @@ -32,6 +32,8 @@ kernelspec: ```{note} This notebook is not intended for use of QR-1 data. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. +Section 5.2 shows how to obtain the ePSF for QR2 images, for users who are still analyzing them. ``` +++ @@ -296,7 +298,7 @@ We advice the users **not** to use the older PSFs, which are stored in the `PSF` Since in these older versions the ePSFs are not provided in the FITS images, we have to obtain them from the respective calibration products. ```{note} -Eventually, after the reprocessing of the data products, the older version LVF images will be replaced by newer versions. Generally, we advise to download the newest images (which also contain improved flagging and calibrations). However, note that the effective PSF of the images is unchanged, but the ePSF provides a better modeling of it than the previous PSFs. It is therefore safe to just exchange the PSFs and use the new ePSF for the older version images. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2 (the newer images also contain improved flagging and calibrations). For users who are still analyzing QR2 images, note that the effective PSF of the images is unchanged, but the ePSF provides a better modeling of it than the previous PSFs. It is therefore safe to just exchange the PSFs and use the new ePSF for the older version images. ``` For this, we first have to figure out the detector on which the image was taken, because the PSF depends on the wavelength (hence detector). diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index 6152120f..8fb11656 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -211,7 +211,7 @@ results = Irsa.query_sia(pos=(coord, search_radius), collection=sia_collection) results_summary(results) ``` -Note that we have used the collection `spherex_dr1` in this case. All the available collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis). +Note that we have used the collection `spherex_dr1` in this case. QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. All the available collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis). +++ ```{tip} From 36012f6c75b65c7a708045232bb2562ecc50801b Mon Sep 17 00:00:00 2001 From: Jessica Krick Date: Thu, 5 Nov 2026 16:01:00 -0500 Subject: [PATCH 22/22] Say IRSA searches no longer return QR2 in the PSF notebook Co-Authored-By: Claude Opus 5.5 --- tutorials/spherex/spherex_psf.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tutorials/spherex/spherex_psf.md b/tutorials/spherex/spherex_psf.md index c9d487aa..e26924ae 100644 --- a/tutorials/spherex/spherex_psf.md +++ b/tutorials/spherex/spherex_psf.md @@ -33,7 +33,8 @@ kernelspec: ```{note} This notebook is not intended for use of QR-1 data. QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. -Section 5.2 shows how to obtain the ePSF for QR2 images, for users who are still analyzing them. +IRSA searches, including the one in this notebook, no longer return QR2 images. +If you already have QR2 images and prefer not to switch to DR1, Section 5.2 shows how to obtain the ePSF for them. ``` +++ @@ -255,7 +256,7 @@ We will use these coordinates later to obtain the correct spatial PSF. ## 5. Obtain the ePSF Depending on Image Version -The PSFs included in the SPHEREx LVF images has been updated. Some of the images therefore still include the old PSF (referred to as "PSF"), while newer versions of the images include the new PSF (referred to as "ePSF"). Differently than the previously used PSF, the ePSF is *not* an optical PSF. Instead, the ePSF is a subpixel-resolved model of the pixel-convolved PSF, which predicts pixel values for a point source at arbitrary sub-pixel positions. Additionally, we implemented a simpler way to read the PSF layer - it is now simply a binary table with array columns. +The PSFs included in the SPHEREx LVF images has been updated. Older images therefore still include the old PSF (referred to as "PSF"), while newer versions of the images include the new PSF (referred to as "ePSF"). Differently than the previously used PSF, the ePSF is *not* an optical PSF. Instead, the ePSF is a subpixel-resolved model of the pixel-convolved PSF, which predicts pixel values for a point source at arbitrary sub-pixel positions. Additionally, we implemented a simpler way to read the PSF layer - it is now simply a binary table with array columns. The differentiation is as follows: * Versions earlier than `7.0` contain the old PSF (saved in the `PSF` FITS layer). These are QR-1 and QR-2 data. @@ -298,7 +299,7 @@ We advice the users **not** to use the older PSFs, which are stored in the `PSF` Since in these older versions the ePSFs are not provided in the FITS images, we have to obtain them from the respective calibration products. ```{note} -QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2 (the newer images also contain improved flagging and calibrations). For users who are still analyzing QR2 images, note that the effective PSF of the images is unchanged, but the ePSF provides a better modeling of it than the previous PSFs. It is therefore safe to just exchange the PSFs and use the new ePSF for the older version images. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2 (the newer images also contain improved flagging and calibrations). IRSA searches no longer return QR2 images, so the query in Section 4.1 finds only DR1 images and the code cells in this section are skipped. If you already have QR2 images and prefer not to switch to DR1, note that the effective PSF of the images is unchanged, but the ePSF provides a better modeling of it than the previous PSFs. It is therefore safe to just exchange the PSFs and use the new ePSF for the older version images. To do this, set `spectral_image_url` in Section 4.1 to your QR2 image and rerun the notebook from there. ``` For this, we first have to figure out the detector on which the image was taken, because the PSF depends on the wavelength (hence detector).