diff --git a/.gitignore b/.gitignore index 3f7c7834..2ad0eb0e 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,10 @@ _build/ #pycache __pycache__ +# Local Python virtual environments +.venv/ +venv/ + # Local working notes, not part of the tutorials CLAUDE.md OU_PR_REVIEW.md diff --git a/tutorials/spherex/spherex.md b/tutorials/spherex/spherex.md index 1f9ea93b..ad4db281 100644 --- a/tutorials/spherex/spherex.md +++ b/tutorials/spherex/spherex.md @@ -4,7 +4,8 @@ SPHEREx observes the sky from roughly 0.75–5.0 µm using a single instrument that provides low-resolution spectroscopy (R ≈ 40–150) in hundreds of spectral channels for every point on the sky. Its science goals span cosmology, galaxy evolution, and the interstellar medium, enabling measurements of large-scale structure, the cosmic history of star formation, and the distribution of key molecules and ices in the Milky Way and nearby galaxies. -SPHEREx data releases include weekly [Quick Release spectral image products](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-products/) (multi-extension FITS files containing calibrated near-infrared surface brightness, variance, flags, modeled backgrounds, PSFs, and wavelength WCS) along with ancillary calibration and metadata files such as gain matrices, dark current maps, solid angle pixel maps, and detailed spectral WCS products for each detector. +SPHEREx data releases include weekly [Quick Release spectral image products](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-products/) as well as periodic Data Releases (e.g., DR1) (multi-extension FITS files containing calibrated near-infrared surface brightness, variance, flags, modeled backgrounds, PSFs, and wavelength WCS) along with ancillary calibration and metadata files such as gain matrices, dark current maps, solid angle pixel maps, and detailed spectral WCS products for each detector. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. ```{notebook-gallery} notebook_metadata.yml tutorials/spherex/spherex_intro.md diff --git a/tutorials/spherex/spherex_cutouts.md b/tutorials/spherex/spherex_cutouts.md index 44c37db9..b9d90549 100644 --- a/tutorials/spherex/spherex_cutouts.md +++ b/tutorials/spherex/spherex_cutouts.md @@ -38,7 +38,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). ## 3. Imports @@ -104,21 +104,33 @@ output_filename = 'spherex_cutouts_mef.fits' Here we show how to use `astroquery`'s IRSA TAP interface to retrieve all images that overlap with the position defined above. This query will retrieve a table of URLs that link to the MEF cutouts. Each row in the table corresponds to a single cutout and includes the data access URL and an observation timestamp. +The SPHEREx metadata tables include images from every release. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2; this query keeps only DR1 images. The results are sorted from oldest to newest. ```{code-cell} ipython3 +# TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and +# revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +# Note: `Irsa.tap_url` must be set directly -- `astroquery.ipac.irsa.conf.tap_url` is only read +# once when the module is first imported, so changing it afterward has no effect. +IRSA_HOST = "https://irsatest.ipac.caltech.edu" +Irsa.tap_url = f"{IRSA_HOST}/TAP" + # Define a query that will search the appropriate SPHEREx metadata tables # for spectral images that cover the chosen coordinates and match the # specified bandpass. Return the cutout data access URL and the time of observation. # Sort by observation time. +# spherex.plane.datarelease is a date, not a release name, so DR1 rows are +# selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - 'https://irsa.ipac.caltech.edu/' || a.uri || '?center={coord.ra.deg},{coord.dec.deg}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={coord.ra.deg},{coord.dec.deg}d&size={size.value}' AS uri, p.time_bounds_lower FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid WHERE 1 = CONTAINS(POINT('ICRS', {coord.ra.deg}, {coord.dec.deg}), p.poly) AND p.energy_bandpassname = '{bandpass}' + AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ @@ -166,7 +178,7 @@ def process_cutout(row, coord, cache): with fits.open(row["uri"], cache=cache) as hdulist: # There are seven HDUs: # 0 contains minimal metadata in the header and no data. - # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, PSF, WCS-WAVE + # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE header = hdulist[1].header # Compute pixel coordinates corresponding to cutout position. diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index 1bad9159..b5714181 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -44,7 +44,8 @@ The community will also mine SPHEREx data and combine it with synergistic data s More information about SPHEREx, the data releases, and the data products is available in the [SPHEREx Archive at IRSA User Guide](https://caltech-ipac.github.io/spherex-archive-documentation). -This notebook can be used with both Quick Release 3 (QR3) and Quick Release 2 (QR2) data. +This notebook uses data from SPHEREx Data Release 1 (DR1). +QR3 and DR1 supersede the earlier Quick Release 2 (QR2), and SPHEREx recommends using QR3 and DR1 over QR2. +++ @@ -109,15 +110,13 @@ The IRSA SIA collections can be listed using using the ``list_collections`` meth +++ The collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis) -There are currently four collections available: +The DR1 Spectral Image MEFs are available in two collections: -* `'spherex_qr3'` -- Quick Release 3 Spectral Image MEFs that are part of the SPHEREx **Wide Survey** -* `'spherex_qr3_deep'` -- Quick Release 3 Spectral Image MEFs that are part of the SPHEREx **Deep Survey** -* `'spherex_qr2'` -- Quick Release 2 Spectral Image MEFs that are part of the SPHEREx **Wide Survey** -* `'spherex_qr2_deep'` -- Quick Release 2 Spectral Image MEFs that are part of the SPHEREx **Deep Survey** +* `'spherex_dr1'` -- DR1 Spectral Image MEFs that are part of the SPHEREx **Wide Survey** +* `'spherex_dr1_deep'` -- DR1 Spectral Image MEFs that are part of the SPHEREx **Deep Survey** ```{code-cell} ipython3 -results = Irsa.query_sia(pos=(coord, search_radius), collection='spherex_qr2') +results = Irsa.query_sia(pos=(coord, search_radius), collection='spherex_dr1') ``` :::{note} @@ -188,7 +187,7 @@ You can see that the Level 2 Spectral Image files are multi-extension FITS files * **FLAGS:** A bitmap with per-pixel status and processing flags. * **VARIANCE:** A per-pixel estimate of the variance. * **ZODI:** An model of the zodiacal dust background signal. Note that this is not subtracted from the IMAGE extension. -* **PSF:** An image cube (3D array) where each plane represents a Point Spread Function (PSF) for a cutout in over-sampled pixel space. +* **EPSF:** A table of effective Point Spread Functions (ePSFs), one for each of 441 regions of the detector, in over-sampled pixel space. * **WCS-WAVE:** Spectral WCS lookup table that maps pixel coordinates to the central wavelength and bandwidth of each pixel. We will tour each extension in the sections below. @@ -362,7 +361,7 @@ Now let's take a look at some header keywords that provide information about how spectral_image_header['L2 N_*'] ``` -There are 14 flags in total. Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The remaining flags are described in Table 8 of the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +These keywords count the pixels carrying most of the flags defined in the FLAGS header (see section 9), plus a few combined counts: keywords ending in `_NB` count pixels that are not also flagged as bad, and those ending in `_NS` count pixels that are not also flagged SOURCE. Typically, most of the pixels are identified as SOURCE pixels, which are pixels mapped to a known source. The flags are described in Table 16 of the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ @@ -382,10 +381,12 @@ for keyword in keywords_to_print: print(f"{keyword}: {value}") ``` -Conveniently, the definitions of the flags are also provided in the FLAGS header. +Conveniently, the definitions of the flags are also provided in the FLAGS header. For each bit number n, the keyword `MSKNnnnn` gives the flag name and `MSKDnnnn` gives its description. ```{code-cell} ipython3 -flags_header['MP*'] +for key in flags_header['MSKN*']: + bit = key[4:] + print(f"{int(bit):2d} {flags_header[key]:<13} {flags_header['MSKD' + bit]}") ``` Tip: In the Firefly visualization, if you are looking at the first extension (IMAGE), you can open up the layers icon (top right) to enable overlaying a visualization of the flags. @@ -420,14 +421,14 @@ for keyword in keywords_to_print: print(f"{keyword}: {value}") ``` -## 12. Explore the fifth extension: PSF +## 12. Explore the fifth extension: EPSF -The fifth extension of the MEF contains 121 Point-spread functions (PSFs); each PSF is represented as a 101 x 101 image and all 121 are assembled together into a cube. Each of the 121 layers represents a "super-resolution" PSF estimate in a different region (defined by an 11x11 grid) of the detector. Each PSF is a two-dimensional array with size of 101 × 101 pixels. The PSFs are oversampled such that 10 PSF pixels cover the same spatial extent as one spectral image pixel (0.615 arcsec). Let's look at some specific header keywords for the PSF: +The fifth extension of the MEF contains the effective Point Spread Function (ePSF), stored as a binary table. The detector is divided into a 21 x 21 grid of regions (each about 97 x 97 spectral image pixels), and each of the 441 rows of the table holds the ePSF for one region, along with that region's position on the detector (`XCENTER`, `YCENTER`) and its central wavelength and bandwidth (`CWAVE`, `CBAND`). Each ePSF is a 33 x 33 array, oversampled by a factor of 5 relative to the spectral image pixels (header keywords `OVSMPX` and `OVSMPY`). Unlike an optical PSF, the ePSF is a model of the pixel-convolved PSF: it predicts the pixel values of a point source at any sub-pixel position. The [SPHEREx ePSF tutorial](spherex_psf.md) shows how to find and use the ePSF for a given position. Let's look at some specific header keywords for the EPSF: ```{code-cell} ipython3 -psf_header = hdulist[5].header +psf_header = hdulist["EPSF"].header -keywords_to_print = ['EXTNAME', 'NAXIS1', 'NAXIS2', 'NAXIS3', 'BUNIT'] +keywords_to_print = ['EXTNAME', 'NAXIS2', 'DETECTOR', 'OVSMPX', 'OVSMPY'] for keyword in keywords_to_print: value = psf_header.get(keyword, 'Keyword not found') diff --git a/tutorials/spherex/spherex_mosaic.md b/tutorials/spherex/spherex_mosaic.md index 3e3a451a..08c68a3c 100644 --- a/tutorials/spherex/spherex_mosaic.md +++ b/tutorials/spherex/spherex_mosaic.md @@ -42,7 +42,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ diff --git a/tutorials/spherex/spherex_psf.md b/tutorials/spherex/spherex_psf.md index f629d475..e26924ae 100644 --- a/tutorials/spherex/spherex_psf.md +++ b/tutorials/spherex/spherex_psf.md @@ -32,6 +32,9 @@ kernelspec: ```{note} This notebook is not intended for use of QR-1 data. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. +IRSA searches, including the one in this notebook, no longer return QR2 images. +If you already have QR2 images and prefer not to switch to DR1, Section 5.2 shows how to obtain the ePSF for them. ``` +++ @@ -48,7 +51,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ @@ -115,17 +118,27 @@ size = 0.01 * u.degree Once we defined the coordinates of interest and the size of the cutout, we run a TAP query to gather all SPHEREx spectral images that cover the coordinates. ```{code-cell} ipython3 +# TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and +# revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +# Note: `Irsa.tap_url` must be set directly -- `astroquery.ipac.irsa.conf.tap_url` is only read +# once when the module is first imported, so changing it afterward has no effect. +IRSA_HOST = "https://irsatest.ipac.caltech.edu" +Irsa.tap_url = f"{IRSA_HOST}/TAP" + # Define a query that will search the appropriate SPHEREx metadata tables # for spectral images that cover the chosen coordinates of interest. # Return the cutout data access URL and the time of observation. # Sort by observation time. +# spherex.plane.datarelease is a date, not a release name, so DR1 rows are +# selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - 'https://irsa.ipac.caltech.edu/' || a.uri || '?center={coord.ra.deg},{coord.dec.deg}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={coord.ra.deg},{coord.dec.deg}d&size={size.value}' AS uri, p.time_bounds_lower, p.energy_bandpassname, p.time_bounds_lower, p.time_bounds_upper, p.provenance_version FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid WHERE 1 = CONTAINS(POINT('ICRS', {coord.ra.deg}, {coord.dec.deg}), p.poly) + AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ @@ -180,7 +193,7 @@ for attempt in range(max_retries): hdul.info() ``` -The downloaded SPHEREx image cutout contains 5 FITS layers, which are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +The downloaded SPHEREx image cutout contains a primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE), which are described in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). Let's also get the version of the image. The version is needed later to decide how to handle the PSF. For obtaining the version of the image, we define a handy function. @@ -243,7 +256,7 @@ We will use these coordinates later to obtain the correct spatial PSF. ## 5. Obtain the ePSF Depending on Image Version -The PSFs included in the SPHEREx LVF images has been updated. Some of the images therefore still include the old PSF (referred to as "PSF"), while newer versions of the images include the new PSF (referred to as "ePSF"). Differently than the previously used PSF, the ePSF is *not* an optical PSF. Instead, the ePSF is a subpixel-resolved model of the pixel-convolved PSF, which predicts pixel values for a point source at arbitrary sub-pixel positions. Additionally, we implemented a simpler way to read the PSF layer - it is now simply a binary table with array columns. +The PSFs included in the SPHEREx LVF images has been updated. Older images therefore still include the old PSF (referred to as "PSF"), while newer versions of the images include the new PSF (referred to as "ePSF"). Differently than the previously used PSF, the ePSF is *not* an optical PSF. Instead, the ePSF is a subpixel-resolved model of the pixel-convolved PSF, which predicts pixel values for a point source at arbitrary sub-pixel positions. Additionally, we implemented a simpler way to read the PSF layer - it is now simply a binary table with array columns. The differentiation is as follows: * Versions earlier than `7.0` contain the old PSF (saved in the `PSF` FITS layer). These are QR-1 and QR-2 data. @@ -267,12 +280,14 @@ Note that the ePSFs are stored in the form of a binary table, which makes it str try: epsf_hdu = image_hdul["EPSF"] except KeyError: + epsf_in_image = False print( "This image does not contain the EPSF extension. " "It was likely created with an older version. " - "Please proceed to Section 5.2." + "Proceeding to Section 5.2." ) else: + epsf_in_image = True epsf_bintable = Table(epsf_hdu.data) epsf_header = epsf_hdu.header print("ePSF loaded.") @@ -284,14 +299,15 @@ We advice the users **not** to use the older PSFs, which are stored in the `PSF` Since in these older versions the ePSFs are not provided in the FITS images, we have to obtain them from the respective calibration products. ```{note} -Eventually, after the reprocessing of the data products, the older version LVF images will be replaced by newer versions. Generally, we advise to download the newest images (which also contain improved flagging and calibrations). However, note that the effective PSF of the images is unchanged, but the ePSF provides a better modeling of it than the previous PSFs. It is therefore safe to just exchange the PSFs and use the new ePSF for the older version images. +QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2 (the newer images also contain improved flagging and calibrations). IRSA searches no longer return QR2 images, so the query in Section 4.1 finds only DR1 images and the code cells in this section are skipped. If you already have QR2 images and prefer not to switch to DR1, note that the effective PSF of the images is unchanged, but the ePSF provides a better modeling of it than the previous PSFs. It is therefore safe to just exchange the PSFs and use the new ePSF for the older version images. To do this, set `spectral_image_url` in Section 4.1 to your QR2 image and rerun the notebook from there. ``` For this, we first have to figure out the detector on which the image was taken, because the PSF depends on the wavelength (hence detector). ```{code-cell} ipython3 -this_detector = image_hdul['IMAGE'].header["DETECTOR"] -print(f"Detector is {this_detector}") +if not epsf_in_image: + this_detector = image_hdul['IMAGE'].header["DETECTOR"] + print(f"Detector is {this_detector}") ``` Next, we load the calibration file for that detector. Note that each detector has one ePSF calibration file as for now we do not assume that the calibration changes as a function of time. @@ -303,37 +319,44 @@ Specifically, the ePSF calibration files are stored in the respective release di We first construct a dictionary linking the detectors with the most current ePSFs in this bucket. ```{code-cell} ipython3 -ePSF_calibs = { - "1": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/1/epsf_D1_spx_cal-epsf-v1-2026-191.fits", - "2": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/2/epsf_D2_spx_cal-epsf-v1-2026-191.fits", - "3": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v2-2026-191/3/epsf_D3_spx_cal-epsf-v2-2026-191.fits", - "4": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/4/epsf_D4_spx_cal-epsf-v1-2026-191.fits", - "5": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/5/epsf_D5_spx_cal-epsf-v1-2026-191.fits", - "6": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/6/epsf_D6_spx_cal-epsf-v1-2026-191.fits" -} +if not epsf_in_image: + ePSF_calibs = { + "1": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/1/epsf_D1_spx_cal-epsf-v1-2026-191.fits", + "2": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/2/epsf_D2_spx_cal-epsf-v1-2026-191.fits", + "3": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v2-2026-191/3/epsf_D3_spx_cal-epsf-v2-2026-191.fits", + "4": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/4/epsf_D4_spx_cal-epsf-v1-2026-191.fits", + "5": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/5/epsf_D5_spx_cal-epsf-v1-2026-191.fits", + "6": "https://nasa-irsa-spherex.s3.us-east-1.amazonaws.com/qr3/epsf/cal-epsf-v1-2026-191/6/epsf_D6_spx_cal-epsf-v1-2026-191.fits" + } ``` -Now we can rad in the correct calibration product ePSF corresponding to the detector in which the image was taken. +Now we can read in the correct calibration product ePSF corresponding to the detector in which the image was taken. ```{code-cell} ipython3 -with fits.open(ePSF_calibs[str(this_detector)]) as hdul: - hdul.info() - epsf_bintable = Table(hdul['EPSF'].data) - epsf_header = hdul['EPSF'].header - epsf_mosaic = hdul['EPSF_MOSAIC'].data +if not epsf_in_image: + with fits.open(ePSF_calibs[str(this_detector)]) as hdul: + hdul.info() + epsf_bintable = Table(hdul['EPSF'].data) + epsf_header = hdul['EPSF'].header + epsf_mosaic = hdul['EPSF_MOSAIC'].data +else: + print("ePSF was already loaded from the image in Section 5.1; skipping the calibration-file fallback.") ``` The calibration product contains the same `EPSF` extension as the SPHEREx LVF image files. In addition, the calibration products contains an `EPSF_MOSAIC` layer, which is *not* available in the LVF images. The ePSF mosaic allows a quick visualization of all the ePSFs on the detector. ```{code-cell} ipython3 -fig = plt.figure(figsize=(16,10)) -ax1 = fig.add_subplot(1,2,1) - -im1 = ax1.imshow(epsf_mosaic, origin='lower') -ax1.set_xlabel("$x$") -ax1.set_ylabel("$y$") -plt.show() +if not epsf_in_image: + fig = plt.figure(figsize=(16,10)) + ax1 = fig.add_subplot(1,2,1) + + im1 = ax1.imshow(epsf_mosaic, origin='lower') + ax1.set_xlabel("$x$") + ax1.set_ylabel("$y$") + plt.show() +else: + print("EPSF_MOSAIC is only available in the calibration product (Section 5.2), not in the LVF image itself.") ``` ```{note} diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py index 71fdc704..e3fd9e0e 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool/aperture_photometry.py @@ -393,7 +393,11 @@ def build_phot_table(results_table: Table, matched_stars: pd.DataFrame, aperture var_img = hdus[hdu_idx_map["VARIANCE"]].data flags = hdus[hdu_idx_map["FLAGS"]].data flags_header = hdus[hdu_idx_map["FLAGS"]].header - flag_dict = {key[3:]: flags_header[key] for key in flags_header if key.startswith('MP_')} + # Map flag names to bit numbers. QR3 and later name bit n in MSKNnnnn; + # QR2 used MP_ = n. + flag_dict = {flags_header[key]: int(key[4:]) for key in flags_header if key.startswith('MSKN')} + if not flag_dict: + flag_dict = {key[3:]: flags_header[key] for key in flags_header if key.startswith('MP_')} flag_dict = dict(sorted(flag_dict.items(), key=lambda x: x[1])) flag_names = list(flag_dict.keys()) flag_masks = {name: (flags & (1 << flag_dict[name])) > 0 for name in flag_names} @@ -531,7 +535,8 @@ def grab_star(obs_log_entry: list[dict], ap_rad: float, bad_flags: list[str]) -> ap_rad : `float` Aperture radius to consider. bad_flags : `list` of `str` - List of bad photometry flags to check for. + List of bad photometry flags to check for. Flags not defined in an + image's FLAGS header are skipped for that image. Returns @@ -566,7 +571,8 @@ def grab_star(obs_log_entry: list[dict], ap_rad: float, bad_flags: list[str]) -> # Sum number of bad pixels in aperture bad_pixels = 0 for flag in bad_flags: - bad_pixels += obs['dqf_sum'][flag][idx_ap] + if flag in obs['dqf_sum']: + bad_pixels += obs['dqf_sum'][flag][idx_ap] bad_flags_aperture.append(bad_pixels) # Filter out non-finite flux measurements and convert to arrays diff --git a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md index 442b36c3..8fb11656 100644 --- a/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md +++ b/tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md @@ -47,7 +47,7 @@ SPHEREx data will be used to: The community will also mine SPHEREx data and combine it with synergistic data sets to address a variety of additional topics in astrophysics. -More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp_QR.pdf). +More information is available in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf). +++ @@ -170,20 +170,20 @@ output_notebook(resources=INLINE) ## 5. Initialize Ephemeral Query Parameters ```{attention} -The query parameters below are valid as of **February 2026**, but will need to be updated as the mission proceeds. +The query parameters below are valid as of **SPHEREx Data Release 1 (DR1), November 2026**, but will need to be updated as the mission proceeds. Specifically, the ephemeral query parameters _will_ change for future SPHEREx data releases. ``` ```{code-cell} ipython3 # General -data_release = "qr2" +data_release = "dr1" # Spectral image queries sia_collection = f"spherex_{data_release}" # Solid angle pixel map queries dataset_type="solid_angle_pixel_map" -sapm_collection = "cal-sapm-v2-2025-164" +sapm_collection = "cal-sapm-v3-2026-191" sapm_s3 = [ f"nasa-irsa-spherex/{data_release}/{dataset_type}/{sapm_collection}/{k}/"\ f"{dataset_type}_D{k}_spx_{sapm_collection}.fits" for k in range(1, 7) @@ -211,7 +211,7 @@ results = Irsa.query_sia(pos=(coord, search_radius), collection=sia_collection) results_summary(results) ``` -Note that we have used the collection `spherex_qr2` in this case. All the available collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis). +Note that we have used the collection `spherex_dr1` in this case. QR3 and DR1 supersede QR2, and SPHEREx recommends using QR3 and DR1 over QR2. All the available collections are documented at [SPHEREx Data Access: Application Program Interfaces (APIs)](https://caltech-ipac.github.io/spherex-archive-documentation/spherex-data-access#application-program-interfaces-apis). +++ ```{tip} @@ -270,7 +270,10 @@ d2_result = results[results['energy_bandpassname'] == 'SPHEREx-D2'][0] # first # Using `astropy.table.TableGroups` results_by_detector = results.group_by('energy_bandpassname') print(results_by_detector.groups.keys) -d4_result = results_by_detector.groups[3][0] # first D4 image +# Select the group by name: not every detector has images at every position, +# so the D4 group is not always at the same index +d4_group = results_by_detector.groups.keys['energy_bandpassname'] == 'SPHEREx-D4' +d4_result = results_by_detector.groups[d4_group][0] # first D4 image ``` ## 7. Load the Data @@ -392,7 +395,7 @@ mask2 = np.zeros(flags_data2.shape) * np.nan mask2[(flags_data2 == 2097152) | (flags_data2 == 2) | (flags_data2 == 0)] = 1 mask4 = np.zeros(flags_data4.shape) * np.nan -mask4[(flags_data4 == 2097152) | (flags_data2 == 2) | (flags_data4 == 0)] = 1 +mask4[(flags_data4 == 2097152) | (flags_data4 == 2) | (flags_data4 == 0)] = 1 ``` Finally, we reproject the images and masks to the same pixel grid. We reproject D2 on D4. For this we use `reproject_exact()` (for flux conservation) and `reproject_interp()` (for mask) from the _reproject_ Python package. @@ -666,9 +669,13 @@ Note that the following procedure follows the one shown in the SPHEREx [Image Cu ``` ```{code-cell} ipython3 +# TESTING ONLY -- DR1 is not yet on production. Point at irsatest for now and +# revert IRSA_HOST to "https://irsa.ipac.caltech.edu" before merging at DR1 release. +IRSA_HOST = "https://irsatest.ipac.caltech.edu" + # Define the service endpoint for IRSA's Table Access Protocol (TAP) # so that we can query SPHEREx metadata tables. -service = pyvo.dal.TAPService("https://irsa.ipac.caltech.edu/TAP") +service = pyvo.dal.TAPService(f"{IRSA_HOST}/TAP") # Specify cutout size (approximately 60 pixels on a side) size = 0.1 * u.degree @@ -677,13 +684,16 @@ size = 0.1 * u.degree # for spectral images that cover the chosen coordinates and match the # specified bandpass. Return the cutout data access URL and the time of observation. # Sort by observation time. +# spherex.plane.datarelease is a date, not a release name, so DR1 rows are +# selected by matching the DR1 path in the artifact URI instead. query = f""" SELECT - 'https://irsa.ipac.caltech.edu/' || a.uri || '?center={matched_stars['ra'][0]},{matched_stars['dec'][0]}d&size={size.value}' AS uri, + '{IRSA_HOST}/' || a.uri || '?center={matched_stars['ra'][0]},{matched_stars['dec'][0]}d&size={size.value}' AS uri, p.time_bounds_lower FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid WHERE 1 = CONTAINS(POINT('ICRS', {matched_stars['ra'][0]}, {matched_stars['dec'][0]}), p.poly) + AND a.uri LIKE 'ibe/data/spherex/dr1/%' ORDER BY p.time_bounds_lower """ @@ -721,7 +731,7 @@ def process_cutout(row, ra, dec, cache, retries=3, delay=5): with fits.open(row["uri"]) as hdulist: # There are seven HDUs: # 0 contains minimal metadata in the header and no data. - # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, PSF, WCS-WAVE + # 1 through 6 are: IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE header = hdulist[1].header # Fetch spatial and spectral WCS @@ -812,11 +822,13 @@ if os.path.exists(phot_path): ``` Apply flagging to the extracted 1D-Spectrum. +We use the flags that set the conservative CONTAINS_BAD_PIXEL warning in the [SPHEREx Explanatory Supplement](https://irsa.ipac.caltech.edu/data/SPHEREx/docs/SPHEREx_Expsupp.pdf) (flag definitions are in its Table 16). ```{code-cell} ipython3 # Define bad photometry flags -bad_flags = ['TRANSIENT', 'OVERFLOW', 'SUR_ERROR', 'PHANTOM', 'REFERENCE', 'NONFUNC', 'DICHROIC', - 'MISSING_DATA', 'HOT', 'COLD', 'FULLSAMPLE', 'PHANMISS', 'NONLINEAR', 'PERSIST', 'OUTLIER'] +bad_flags = ['TRANSIENT', 'OVERFLOW', 'SUR_ERROR', 'NONFUNC', 'DICHROIC', 'MISSING_DATA', 'HOT', 'COLD', + 'NONLINEAR', 'CROSSTALK', 'GHOST', 'GHOST_FPA', 'GHOST_EXT', 'STREAK', 'PERSIST', + 'BLOOM', 'SNOWBALL', 'HALO', 'SATELLITE_HALO'] ap_rad = 3.0 # set your preferred aperture radius, from the `aperture_radii` list you specified above targ = 0 # pick index from `all_stars_band_phot` table wvl, bdw, abmags, abmag_errs, flx, var, n_bad = \