From a7d3c75c69beac41f72997b2d94dfd85c0f6e11e Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 28 Oct 2026 13:42:17 -0700 Subject: [PATCH 1/4] Add a draft skill for SPHEREx data-release notebook updates Captures the DR1 update as a repeatable workflow: release-dependent spots in each notebook, settled decisions and gotchas, a release-availability check script, and a parameterized Source Discovery Tool test harness. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/spherex-data-release/SKILL.md | 118 ++++++++++++++ .../references/decisions.md | 57 +++++++ .../references/touchpoints.md | 77 +++++++++ .../scripts/check_release.sh | 55 +++++++ .../scripts/run_sdt_release_test.py | 146 ++++++++++++++++++ 5 files changed, 453 insertions(+) create mode 100644 .claude/skills/spherex-data-release/SKILL.md create mode 100644 .claude/skills/spherex-data-release/references/decisions.md create mode 100644 .claude/skills/spherex-data-release/references/touchpoints.md create mode 100755 .claude/skills/spherex-data-release/scripts/check_release.sh create mode 100644 .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py diff --git a/.claude/skills/spherex-data-release/SKILL.md b/.claude/skills/spherex-data-release/SKILL.md new file mode 100644 index 00000000..69daff10 --- /dev/null +++ b/.claude/skills/spherex-data-release/SKILL.md @@ -0,0 +1,118 @@ +--- +name: spherex-data-release +description: Update the SPHEREx tutorial notebooks in irsa-tutorials for a new SPHEREx data release (DR2, DR3, a new Quick Release, a reprocessing). Use this whenever someone mentions a new SPHEREx release, new SPHEREx collections or DOIs, a new SPHEREx Explanatory Supplement, new FLAGS bits, or asks to "update the SPHEREx notebooks", "test SPHEREx against irsatest", or prepare the SPHEREx release-day PR -- even if they only ask about one of those pieces. +--- + +# SPHEREx data-release update + +This skill repeats the work done for DR1 (issue #392, released 2026-11-12): find every place in the +SPHEREx notebooks that depends on the release, test against pre-release IRSA hosts, collect the +answers that only other people can give, and land everything in one PR on release day. + +Most of the time goes into waiting on other people, not editing code. So the main job is to keep a +written plan current, ask for missing information early, and do everything that does not depend on +those answers right away. + +## Files in this skill + +- `references/touchpoints.md` -- every release-dependent spot in each notebook, with what it looked + like for DR1. Read this first; it is the work list. +- `references/decisions.md` -- rules settled during DR1 (which release to show, which DOI to cite, + which flags count as bad, who owns which notebook) and gotchas that cost time. Read before + asking the user anything; most questions were already answered once. +- `scripts/check_release.sh ` -- what is live for release `` (e.g. `dr2`) on the + pre-release and production hosts, S3 buckets and docs. +- `scripts/run_sdt_release_test.py` -- runs the Source Discovery Tool's own notebook cells against + pre-release data, with stand-ins for the parts that are not live yet. + +## Workflow + +### 1. Start a plan file + +Create `SPHEREX__PLAN.md` at the repo root and add it (and any local test folder) to +`.git/info/exclude` so it never gets committed. Put a short "RESUME HERE" block at the top: branch, +last commit, what is blocked and on whom, next actions. Update it at the end of every work chunk. +The work spans weeks and many sessions, and this file is the only thing that carries over. + +Work on a branch named `spherex-` off `main`. Push it to the user's fork only until release; +the test-host settings must not reach `main` early. + +### 2. Find out what exists + +Run `scripts/check_release.sh ` (needs IRSA VPN for the pre-release hosts). It reports, for +TAP, SIA, IBE, S3 and the supplement PDF, whether the new release is visible. Record the results as +a "data products needed" table in the plan (product, which notebook uses it, test location, +available yes/no, date checked). Re-run it whenever something might have changed. + +### 3. Ask early for what only people can provide + +Send these questions as soon as the release date is known. Draft each as a paste-ready message +in a fenced code block. Keep each one short and specific. + +- **IRSA** (archive/backend staff): Will the new SIA collections be loaded on the pre-release host + (irsadev) before release, so the SIA notebooks can be tested? What will the collections be + named? Will `cloud_access` be filled in those rows? (On irsadev for DR1 it was empty for QR3 + rows; the Source Discovery Tool reads its S3 paths from it.) Will the data be on the staging + S3 bucket, and when does it move to the public one? +- **IRSA** (DOIs): the DOI for the new release. It is usually not available until close to release. +- **SSDC**: a copy of the new Explanatory Supplement, or confirmation that the online one is + current. Ask about any new FLAGS bits and which ones should count as bad for photometry. + +Then decide with the user what happens if the pre-release SIA load does not come through +(for DR1 the fallback was letting the tutorials go out a few days after release). Set a date to +follow up and write it in the plan. + +### 4. Make the changes that do not depend on answers + +Go through `references/touchpoints.md` notebook by notebook. For each one, make the change, then +test it against the pre-release host. In order of how early they can be done: + +1. TAP notebooks (`spherex_cutouts.md`, `spherex_psf.md`, SDT sec. 12): point at the pre-release + host through an `IRSA_HOST` variable inside a clearly marked `# TESTING ONLY` block, and add the + release filter on `a.uri`. Run them end to end with + `jupytext --to notebook --execute `. +2. Header-driven code and text: open one new-release image and compare its extensions, FLAGS + `MSKN*` keywords and `L2 N_*` counts with what the notebooks say and parse (see + touchpoints). New flags or renamed keywords break code silently -- the DR1 photometry recorded + no flags at all until the parser was fixed. +3. SIA notebooks (`spherex_intro.md`, SDT secs. 6-11): switch to the guessed collection names now + (pattern below), so only a check is left for release day. +4. Narrative: supplement link and table number, DOIs, release wording on the landing page. + +Collection names follow the pattern of the previous release: for DR1 they were guessed as +`spherex_dr1`, `spherex_dr1_deep`, `spherex_dr1_cal` from `spherex_qr3`, `spherex_qr3_deep`, +`spherex_qr3_cal`. Put a "verify the guessed names" step in the release-day checklist with the +exact lines to change if they differ. + +Commit in small, single-purpose commits and push to the fork. Ask before committing, and say in +the plan what each commit did. + +### 5. Test the Source Discovery Tool + +The SDT needs SIA results and S3 reads that do not exist before release. +`scripts/run_sdt_release_test.py` runs the notebook's own cells with stand-ins: pre-release TAP rows +instead of SIA (with a `cloud_access` value built from the URI), pre-release IBE instead of S3, and +one 2MASS star instead of the SExtractor/interactive selection. Run it from the repo root: + +```bash +.venv/bin/python .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py \ + tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md --release dr2 +``` + +Report the number of spectral points, how many are flagged, and the flux near 2.16 um against +the 2MASS K estimate. For DR1: 281 points, 40 flagged with the final `bad_flags`, about 17.9 mJy. + +### 6. Release day + +Keep a release-day checklist in the plan from the start and refine it as answers arrive. Its +final shape for DR1 is in `references/decisions.md` under "Release-day checklist". Open a draft PR +well before release so it can be reviewed, with the checklist in the PR description. Merge any +pending cleanup PRs into the release branch first so the diff shows only release work. + +As the last checklist item, update this skill with what was learned (see below). + +## Keeping this skill current + +After each release, update `references/touchpoints.md` with the new values and line numbers, add +new decisions and gotchas to `references/decisions.md`, and fix anything in this file that was +wrong. Leave out people's names: say which group to ask (IRSA, SSDC), not who. diff --git a/.claude/skills/spherex-data-release/references/decisions.md b/.claude/skills/spherex-data-release/references/decisions.md new file mode 100644 index 00000000..48f03c9e --- /dev/null +++ b/.claude/skills/spherex-data-release/references/decisions.md @@ -0,0 +1,57 @@ +# Decisions and gotchas from DR1 + +## Rules (settled; don't re-ask unless something changed) + +- **Show only the newest release.** Older collections stay searchable, but the tutorials + demonstrate only the newest one. TAP notebooks filter on `a.uri LIKE 'ibe/data/spherex//%'`. +- **DOIs:** cite only the DOI of each release the notebook actually loads (both if it loads two). +- **`bad_flags` in the Source Discovery Tool:** use the flag list that sets the supplement's + CONTAINS_BAD_PIXEL warning (supplement v2.0, p. 43). This came from the SSDC developer who wrote + the SDT. Do not use the shorter "flags for masking pixels for photometry calculations" list on + the same page. For DR1 the list was: TRANSIENT, OVERFLOW, SUR_ERROR, NONFUNC, DICHROIC, + MISSING_DATA, HOT, COLD, NONLINEAR, CROSSTALK, GHOST, GHOST_FPA, GHOST_EXT, STREAK, PERSIST, + BLOOM, SNOWBALL, HALO, SATELLITE_HALO. FULLSAMPLE is informational and was removed. +- **Supplement:** link `SPHEREx_Expsupp.pdf` (the `_QR` suffix was dropped for DR1). For flag + definitions, the online supplement is the reference -- not plans for a future release. Cite + the flag table by number (Table 16 in v2.0) and recheck the number when a new version appears. +- **Ownership:** IRSA updates the Source Discovery Tool notebook, not SSDC. +- **Test hosts** go in a `# TESTING ONLY` block with an `IRSA_HOST` variable, so release day is a + mechanical revert. Push only to a personal fork until release. +- **`cloud_access`:** assume the new SIA collection fills it. If a pre-release collection has it + empty, build the S3 path from the URI for local testing only, and don't commit that. +- **Demo mosaic cube:** not regenerated per release. + +## Gotchas + +- irsatest/irsadev are reachable only on the IRSA network or VPN. Cloud notebook servers get 403 or + cannot resolve them. +- IRSA production TAP/SIA can return 502 for a while. For non-SPHEREx lookups in tests (like + 2MASS), irsatest works too. +- `spherex.plane.datarelease` is a date. Filter by URI path or `obs_publisher_did`. +- Each release has a `_cal` SIA collection listing calibration files (ePSF, SAPM) per detector -- + the quickest way to learn calibration version names. +- IBE mirrors the S3 paths, so `s3://nasa-irsa-spherex//...` reads can be pointed at + `https:///ibe/data/spherex//...` for testing. +- Pre-release S3 data sat in a staging bucket (`nasa-irsa-spherex-dr1-staging`) that moved to the + public `nasa-irsa-spherex` on release day. +- FLAGS keywords changed between releases (`MP_*` in QR2, `MSKN*`/`MSKD*` from QR3). New flags + can appear in the supplement before they appear in image headers (SATELLITE_HALO, bit 29, was in + supplement v2.0 but not in DR1 v7.0.5 headers) -- code must tolerate missing flag names. +- Expect overlapping cleanup PRs. Merge them into the release branch before opening the release + PR, or the same changes show up twice. + +## Release-day checklist (DR1's final shape) + +1. Confirm the release is live: `scripts/check_release.sh `; verify the guessed SIA collection + names and fix intro + SDT if they differ; confirm production SIA rows have `cloud_access`. +2. Revert test hosts: delete each `# TESTING ONLY` block and `IRSA_HOST`, inline + `https://irsa.ipac.caltech.edu`, and check that + `grep -rn "irsatest\|irsadev\|IRSA_HOST" tutorials/` finds nothing. +3. Put in the new DOI (SDT acknowledgements). +4. Bump `**Updated:**` in every notebook that changed. +5. Run each notebook against production with `jupytext --to notebook --execute` (SDT needs its + conda env, `conda-spherex_sdt.yml`). +6. Re-run `techniques-and-tools/MOST_queries.md` once the release is in MOST. +7. Recheck the supplement's flag table number. +8. Mark the draft PR ready for review (it closes the release issue). +9. Update this skill with what was learned. diff --git a/.claude/skills/spherex-data-release/references/touchpoints.md b/.claude/skills/spherex-data-release/references/touchpoints.md new file mode 100644 index 00000000..0701eb9d --- /dev/null +++ b/.claude/skills/spherex-data-release/references/touchpoints.md @@ -0,0 +1,77 @@ +# Release-dependent spots in the SPHEREx notebooks + +Values shown are what DR1 used (branch `spherex-dr1`, Nov 2026). Line numbers drift; search for the +quoted text instead. Paths are relative to `tutorials/spherex/`. + +## spherex_cutouts.md (TAP + IBE cutouts) + +- `IRSA_HOST` test block (`# TESTING ONLY`): sets `IRSA_HOST` and `Irsa.tap_url`. Note + `Irsa.tap_url` must be set directly; `astroquery.ipac.irsa.conf.tap_url` is read only at import. +- TAP release filter: `AND a.uri LIKE 'ibe/data/spherex/dr1/%'`. `spherex.plane.datarelease` is a + date, not a release name, so filter on the URI path. Without a filter the query mixes releases + and `results[...][0]` picks the oldest. +- Cutout URL prefix built from `{IRSA_HOST}/` || `a.uri`. +- Supplement link (`SPHEREx_Expsupp.pdf`). +- `**Updated:**` date at the bottom. + +## spherex_psf.md (TAP + IBE cutouts + S3) + +- Same `IRSA_HOST` block, release filter and cutout URL prefix as cutouts. +- Sec. 5.1 reads the image's own `EPSF` extension (image version >= 7.0). Sec. 5.2 is a fallback + for older images that reads ePSF calibration files from S3 (`/epsf/cal-epsf-v*-*`). It must + stay guarded so it does not overwrite a new image's own ePSF (a DR1 bug). +- "primary HDU plus 6 extensions (IMAGE, FLAGS, VARIANCE, ZODI, EPSF, WCS-WAVE)" -- check against a + new image. +- Supplement link, `**Updated:**`. + +## spherex_intro.md (SIA) + +- Collection list and the `Irsa.query_sia(..., collection='spherex_dr1')` call; bullet list of + "two collections" (wide `spherex_dr1`, deep `spherex_dr1_deep`). Show only the newest release. +- "This notebook uses data from SPHEREx Data Release 1 (DR1)." +- Sec. 6 extension list and sec. 12 describe the `EPSF` bintable (DR1: 441 rows = 21x21 regions, + 33x33 ePSF, 5x oversampled via `OVSMPX`/`OVSMPY`; columns XCENTER/YCENTER/CWAVE/CBAND). +- Sec. 8d: prints `spectral_image_header['L2 N_*']` and explains it. DR1 had 26 keys: one per flag + for most flags (STREAK has none), `_NB` = not also flagged bad, `_NS` = not also flagged SOURCE, + `N_SAT_HALO` = SATELLITE_HALO. Cites the supplement's flag table (Table 16 in supplement v2.0). +- Sec. 9: loops over FLAGS header `MSKNnnnn` (name) / `MSKDnnnn` (description). QR2 used `MP_*`. + DR1 defined 22 flags. +- Supplement link, `**Updated:**`. + +## spherex_source_discovery/spherex_source_discovery_tool_demo.md (SIA + S3 + TAP) + +- Sec. 5 parameters: `data_release = "dr1"`, `sia_collection = f"spherex_{data_release}"`, + `sapm_collection = "cal-sapm-v3-2026-191"`. `data_release` also builds the S3 path + `nasa-irsa-spherex/{data_release}/solid_angle_pixel_map/{sapm_collection}/...` (folder is + `solid_angle_pixel_map/`, not `sapm/`). If only the SIA name changes, set `sia_collection` + directly. Remove any "valid as of " note that no longer holds. +- "Note that we have used the collection `spherex_dr1`" sentence after the SIA query. +- Sec. 7 `get_s3_fpath(cloud_access)`: needs `cloud_access` filled in SIA results. +- Sec. 8 masks: D2 mask from `flags_data2`, D4 mask from `flags_data4` (DR1 fixed a typo here). +- Sec. 12.1 TAP query: `IRSA_HOST` test block (uses `pyvo.dal.TAPService(f"{IRSA_HOST}/TAP")`), + release filter, cutout URL prefix. +- Sec. 12.2 `bad_flags`: the supplement's CONTAINS_BAD_PIXEL list (see decisions.md). Any listed + flag in the aperture drops the point from the plotted spectrum (`bokeh_viz.plot_spectrum`). +- Acknowledgements: DOI sentence. Cite only the DOIs of releases the notebook actually loads. +- Supplement link, `**Updated:**`. +- Package `spherex_source_discovery_tool/aperture_photometry.py`: + `build_phot_table` reads flag names from `MSKN*` (falls back to `MP_*`); `grab_star` skips + `bad_flags` entries an image does not define. Check both against new FLAGS headers. + +## spherex_mosaic.md + +- Supplement link. The static `M101_irsa.fits` demo cube was not regenerated for DR1. + +## spherex.md (landing page) + +- Wording about Quick Releases and Data Releases. + +## techniques-and-tools/MOST_queries.md + +- No release-specific code (`catalog="spherex"`). Re-run once the release is queryable in MOST to + refresh its outputs. + +## Repo files + +- `deploy_to_fornax_manifest.in` and `notebook_metadata.yml` list the SPHEREx files; update them if + files are added, renamed or removed (a stale manifest entry fails CI). diff --git a/.claude/skills/spherex-data-release/scripts/check_release.sh b/.claude/skills/spherex-data-release/scripts/check_release.sh new file mode 100755 index 00000000..f5550d41 --- /dev/null +++ b/.claude/skills/spherex-data-release/scripts/check_release.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Report what is live for a SPHEREx release on IRSA's pre-release and production hosts. +# Usage: check_release.sh e.g. check_release.sh dr2 +# The irsadev/irsatest checks need the IRSA network or VPN. +set -u +REL="${1:?usage: check_release.sh }" + +sia_collections() { + local q="SELECT DISTINCT obs_collection FROM ivoa.obscore WHERE obs_collection LIKE 'spherex%'" + curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" \ + | tail -n +2 | tr '\n' ' ' +} + +tap_rows() { + local q="SELECT TOP 1 a.uri FROM spherex.artifact a WHERE a.uri LIKE 'ibe/data/spherex/$REL/%'" + curl -s -m 120 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p +} + +cloud_access() { + local q="SELECT TOP 1 cloud_access FROM ivoa.obscore WHERE obs_collection = 'spherex_$REL'" + curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p +} + +# Subfolders under / in a bucket ("(empty)" if none, or the S3 error code). +s3_prefixes() { + local xml out + xml=$(curl -s -m 60 "https://$1.s3.us-east-1.amazonaws.com/?list-type=2&delimiter=/&prefix=$REL/") + if grep -q '' <<<"$xml"; then + grep -o '[^<]*' <<<"$xml" | sed 's/<[^>]*>//g' + return + fi + out=$(grep -o '[^<]*' <<<"$xml" | sed 's/<[^>]*>//g' | tr '\n' ' ') + echo "${out:-(empty)}" +} + +http_code() { curl -s -m 30 -o /dev/null -I -w '%{http_code}' "$1"; } + +for host in https://irsadev.ipac.caltech.edu https://irsatest.ipac.caltech.edu https://irsa.ipac.caltech.edu; do + echo "== $host" + echo " SIA collections: $(sia_collections "$host")" + uri=$(tap_rows "$host") + echo " $REL TAP row: ${uri:-(none)}" + echo " cloud_access (spherex_$REL): $(cloud_access "$host")" + # IBE has no directory listing, so check that the TAP row's file is served. + [ -n "$uri" ] && echo " IBE file for that row: HTTP $(http_code "$host/$uri")" +done + +echo "== S3" +echo " staging nasa-irsa-spherex-$REL-staging: $(s3_prefixes "nasa-irsa-spherex-$REL-staging")" +echo " public nasa-irsa-spherex: $(s3_prefixes nasa-irsa-spherex)" + +echo "== Docs" +for f in SPHEREx_Expsupp.pdf SPHEREx_Expsupp_QR.pdf; do + echo " $f: $(http_code "https://irsa.ipac.caltech.edu/data/SPHEREx/docs/$f")" +done diff --git a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py new file mode 100644 index 00000000..5b0188ef --- /dev/null +++ b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py @@ -0,0 +1,146 @@ +"""Run the Source Discovery Tool notebook's own sec. 5, 7, 8 and 12 cells against a SPHEREx +release that is not yet in SIA or public S3. + +Usage (from the repo root, IRSA network or VPN): + python run_sdt_release_test.py \ + --release dr2 [--host https://irsatest.ipac.caltech.edu] + +Stand-ins: +- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri +- s3://nasa-irsa-spherex//... reads -> the IBE mirror of the same path on --host +- secs. 10-11 (SExtractor + interactive selection) -> one 2MASS star near the field center +Secs. 9-11 (SExtractor, Firefly, bokeh) are not exercised. +""" +import argparse +import os +import sys +import json +import traceback + +import jupytext +import astropy.io.fits as fits_mod + +_ap = argparse.ArgumentParser() +_ap.add_argument("notebook") +_ap.add_argument("--release", required=True, help="e.g. dr2") +_ap.add_argument("--host", default="https://irsatest.ipac.caltech.edu") +ARGS = _ap.parse_args() +REL, HOST = ARGS.release, ARGS.host.rstrip("/") + +NB = ARGS.notebook +SDT_DIR = os.path.dirname(os.path.abspath(NB)) +sys.path.insert(0, SDT_DIR) + +S3_PREFIX = f"s3://nasa-irsa-spherex/{REL}/" +IBE_PREFIX = f"{HOST}/ibe/data/spherex/{REL}/" +_real_open = fits_mod.open + + +def open_via_ibe(name, *args, **kwargs): + if isinstance(name, str) and name.startswith(S3_PREFIX): + kwargs.pop("fsspec_kwargs", None) + name = IBE_PREFIX + name[len(S3_PREFIX):] + return _real_open(name, *args, **kwargs) + + +fits_mod.open = open_via_ibe + +cells = [c["source"] for c in jupytext.read(NB).cells if c["cell_type"] == "code"] + + +def cell(marker): + hits = [c for c in cells if marker in c] + assert len(hits) == 1, (marker, len(hits)) + return hits[0] + + +ns = {"HOST": HOST, "REL": REL} + + +def run(label, src): + print(f"\n===== {label} =====", flush=True) + try: + exec(src, ns) + return True + except Exception: + traceback.print_exc() + return False + + +run("imports", cell("import concurrent.futures")) +run("sec 5 params", cell('data_release = "')) +print("sapm_s3[1] =", ns["sapm_s3"][1]) + +# ---- sec. 6 stand-in: first D2 and D4 images of the release at the SDT position, from TAP +run("sec 6 stand-in", ''' +coord = SkyCoord(150.7817877, 3.3502204, unit='deg') +_svc = pyvo.dal.TAPService(HOST + "/TAP") +_q = """SELECT a.uri, p.energy_bandpassname, p.time_bounds_lower +FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid +WHERE 1 = CONTAINS(POINT('ICRS', 150.7817877, 3.3502204), p.poly) + AND a.uri LIKE 'ibe/data/spherex/%s/%%' +ORDER BY p.time_bounds_lower""" % REL +_t = _svc.search(_q).to_table() +print(len(_t), REL, "images at SDT position;", sorted(set(_t["energy_bandpassname"]))) +def _fake_row(det): + r = _t[_t["energy_bandpassname"] == det][0] + key = r["uri"][len("ibe/data/spherex/"):] + return {"cloud_access": json.dumps({"aws": {"bucket_name": "nasa-irsa-spherex", "key": key, + "region": "us-east-1"}})} +d2_result = _fake_row("SPHEREx-D2") +d4_result = _fake_row("SPHEREx-D4") +''') + +ok7 = run("sec 7 get_s3_fpath", cell("def get_s3_fpath")) and \ + run("sec 7 load images", cell("s3_fpath_d2 = get_s3_fpath")) +if ok7: + print("D2 path:", ns["s3_fpath_d2"]) +ok8 = ok7 and all([ + run("sec 8 SAPM", cell("s3_fpath_sapm2 = sapm_s3[1]")), + run("sec 8 units", cell('print(f"Spectral image BUNIT')), + run("sec 8 background", cell("bkg2 = sep.Background")), + run("sec 8 masks", cell("mask2 = np.zeros")), + run("sec 8 reproject", cell("img_data2_reproj, footprint = reproject_exact")), +]) +if ok8: + import numpy as np + print("D2 BUNIT now:", ns["img_hdr2"]["BUNIT"], "| reprojected finite px:", + int(np.isfinite(ns["img_data2_reproj"]).sum()), + "| good-pixel fraction D2 mask:", float(np.nanmean(ns["mask2"] == 1))) + +# ---- secs. 10-11 stand-in: one 2MASS star near the field center +run("matched_stars stand-in", ''' +import pandas as pd +# 2MASS from --host's TAP (production IRSA TAP can return 502s) +import pyvo as _pyvo +_stars = _pyvo.dal.TAPService(HOST + "/TAP").run_sync( + f"SELECT ra, dec, k_m FROM fp_psc WHERE CONTAINS(POINT('ICRS', ra, dec), " + f"CIRCLE('ICRS', {coord.ra.deg}, {coord.dec.deg}, {50/3600})) = 1").to_table() +_stars = _stars[(_stars["k_m"] > 11) & (_stars["k_m"] < 14)] +_stars.sort("k_m") +matched_stars = pd.DataFrame({"NUMBER": [65], "ra": [float(_stars["ra"][0])], "dec": [float(_stars["dec"][0])]}) +print(matched_stars, "Kmag", _stars["k_m"][0]) +''') + +ok12 = all([ + run("sec 12.1 TAP query", cell("service = pyvo.dal.TAPService")), + run("sec 12.1 process_cutout def", cell("def process_cutout")), + run("sec 12.1 results_table", cell('results_table["cutout_index"]')), + run("sec 12.1 download cutouts", cell("Process cutouts in parallel")), +]) +os.makedirs("data", exist_ok=True) +if ok12: + run("sec 12.2 build_phot_table", cell("build_phot_table(results_table")) + run("sec 12.2 load pickle", cell("with open(phot_path, 'rb')")) + if run("sec 12.2 grab_star", cell("bad_flags = [")): + print("n points:", len(ns["wvl"]), "| wl range:", ns["wvl"].min(), ns["wvl"].max()) + +if "wvl" in ns: + import numpy as np + w, f, nb = np.asarray(ns["wvl"]), np.asarray(ns["flx"]), np.asarray(ns["n_bad"]) + k = np.abs(w - 2.16) < 0.1 + print("flux near 2.16 um (mJy, median):", np.nanmedian(f[k]), "| expected for Kmag", + "~", 666.7e3 * 10 ** (-11.946 / 2.5)) + print("points with >=1 flagged pixel in aperture:", int((nb > 0).sum()), "of", len(nb)) + entry = ns["all_stars_band_phot"]["obs_log"][0][0] + print("flags recorded:", sorted(entry["dqf_sum"].keys())) From 5f671f42e5365c62c831a8e1a950f415400e6d68 Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 28 Oct 2026 17:04:49 -0700 Subject: [PATCH 2/4] Let the SPHEREx release skill test SIA against a pre-release host run_sdt_release_test.py gains --sia-host to run the notebook's own SIA cells against irsadev; decisions.md notes how to test SIA notebooks there. Co-Authored-By: Claude Opus 5.5 --- .../references/decisions.md | 5 +++++ .../scripts/run_sdt_release_test.py | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.claude/skills/spherex-data-release/references/decisions.md b/.claude/skills/spherex-data-release/references/decisions.md index 48f03c9e..abf040b6 100644 --- a/.claude/skills/spherex-data-release/references/decisions.md +++ b/.claude/skills/spherex-data-release/references/decisions.md @@ -37,6 +37,11 @@ - FLAGS keywords changed between releases (`MP_*` in QR2, `MSKN*`/`MSKD*` from QR3). New flags can appear in the supplement before they appear in image headers (SATELLITE_HALO, bit 29, was in supplement v2.0 but not in DR1 v7.0.5 headers) -- code must tolerate missing flag names. +- To test SIA notebooks against irsadev, set `Irsa.sia_url = "https://irsadev.ipac.caltech.edu/SIA"` + in a temporary copy (don't commit it). `Irsa.list_collections()` still lists production + collections, so check irsadev collections with a TAP query on `ivoa.obscore` instead. irsadev SIA + `access_url`s point at a dev IBE host. IRSA may load only part of the sky, so confirm the notebook + positions are covered. - Expect overlapping cleanup PRs. Merge them into the release branch before opening the release PR, or the same changes show up twice. diff --git a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py index 5b0188ef..3f305657 100644 --- a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py +++ b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py @@ -3,10 +3,13 @@ Usage (from the repo root, IRSA network or VPN): python run_sdt_release_test.py \ - --release dr2 [--host https://irsatest.ipac.caltech.edu] + --release dr2 [--host https://irsatest.ipac.caltech.edu] \ + [--sia-host https://irsadev.ipac.caltech.edu] Stand-ins: -- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri +- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri. + With --sia-host, sec. 6 instead runs the notebook's own SIA cells against that host (use this + once IRSA has loaded the release's SIA collections there). - s3://nasa-irsa-spherex//... reads -> the IBE mirror of the same path on --host - secs. 10-11 (SExtractor + interactive selection) -> one 2MASS star near the field center Secs. 9-11 (SExtractor, Firefly, bokeh) are not exercised. @@ -24,6 +27,7 @@ _ap.add_argument("notebook") _ap.add_argument("--release", required=True, help="e.g. dr2") _ap.add_argument("--host", default="https://irsatest.ipac.caltech.edu") +_ap.add_argument("--sia-host", help="run sec. 6 with real SIA from this host") ARGS = _ap.parse_args() REL, HOST = ARGS.release, ARGS.host.rstrip("/") @@ -71,8 +75,17 @@ def run(label, src): run("sec 5 params", cell('data_release = "')) print("sapm_s3[1] =", ns["sapm_s3"][1]) +if ARGS.sia_host: + # ---- sec. 6 for real, against the pre-release SIA host + ns["Irsa"].sia_url = ARGS.sia_host.rstrip("/") + "/SIA" + run("sec 6 coord", cell("search_radius = 50 * u.arcsec")) + run("sec 6 SIA query", cell("results = Irsa.query_sia(")) + run("sec 6 pick D2/D4", cell("d2_result = results[")) + print(len(ns["results"]), "SIA rows; D2 cloud_access:", str(ns["d2_result"]["cloud_access"])[:120]) + # ---- sec. 6 stand-in: first D2 and D4 images of the release at the SDT position, from TAP -run("sec 6 stand-in", ''' +else: + run("sec 6 stand-in", ''' coord = SkyCoord(150.7817877, 3.3502204, unit='deg') _svc = pyvo.dal.TAPService(HOST + "/TAP") _q = """SELECT a.uri, p.energy_bandpassname, p.time_bounds_lower From d7e7347c43bd35406c18259e800fefdac452e069 Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 29 Oct 2026 12:33:16 -0700 Subject: [PATCH 3/4] Check SPHEREx cloud_access through SIA in the release script TAP's ivoa.obscore leaves cloud_access blank even where SIA fills it (production QR3 included), so the script reported a false blank. Ask SIA, as the notebooks do, at the position of one of the release's own images. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/spherex-data-release/SKILL.md | 5 +++-- .../spherex-data-release/scripts/check_release.sh | 13 +++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.claude/skills/spherex-data-release/SKILL.md b/.claude/skills/spherex-data-release/SKILL.md index 69daff10..6395cddb 100644 --- a/.claude/skills/spherex-data-release/SKILL.md +++ b/.claude/skills/spherex-data-release/SKILL.md @@ -51,8 +51,9 @@ in a fenced code block. Keep each one short and specific. - **IRSA** (archive/backend staff): Will the new SIA collections be loaded on the pre-release host (irsadev) before release, so the SIA notebooks can be tested? What will the collections be - named? Will `cloud_access` be filled in those rows? (On irsadev for DR1 it was empty for QR3 - rows; the Source Discovery Tool reads its S3 paths from it.) Will the data be on the staging + named? Will `cloud_access` be filled in those rows? (The Source Discovery Tool reads its S3 + paths from it. Check it in SIA results, e.g. with `scripts/check_release.sh`: TAP's + `ivoa.obscore` leaves it blank even when SIA fills it.) Will the data be on the staging S3 bucket, and when does it move to the public one? - **IRSA** (DOIs): the DOI for the new release. It is usually not available until close to release. - **SSDC**: a copy of the new Explanatory Supplement, or confirmation that the online one is diff --git a/.claude/skills/spherex-data-release/scripts/check_release.sh b/.claude/skills/spherex-data-release/scripts/check_release.sh index f5550d41..11676a59 100755 --- a/.claude/skills/spherex-data-release/scripts/check_release.sh +++ b/.claude/skills/spherex-data-release/scripts/check_release.sh @@ -16,9 +16,18 @@ tap_rows() { curl -s -m 120 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p } +# Ask SIA, not TAP: ivoa.obscore leaves cloud_access blank even where SIA (what the notebooks +# use) fills it. Search at the position of one of the release's own images. IRSA's SIA CSV +# doesn't quote the JSON, so grep it out instead of parsing columns. cloud_access() { - local q="SELECT TOP 1 cloud_access FROM ivoa.obscore WHERE obs_collection = 'spherex_$REL'" - curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p + local q="SELECT TOP 1 s_ra, s_dec FROM ivoa.obscore WHERE obs_collection = 'spherex_$REL'" + local pos ra dec row + pos=$(curl -s -m 60 --data-urlencode "QUERY=$q" -d FORMAT=csv -d LANG=ADQL "$1/TAP/sync" | sed -n 2p) + [ -z "$pos" ] && { echo "(no spherex_$REL rows)"; return; } + IFS=, read -r ra dec <<<"$pos" + row=$(curl -s -m 120 "$1/SIA?COLLECTION=spherex_$REL&POS=CIRCLE+$ra+$dec+0.0003&MAXREC=1&RESPONSEFORMAT=csv" | sed -n 2p) + [ -z "$row" ] && { echo "(no SIA rows)"; return; } + grep -o '{"aws": {[^}]*}}' <<<"$row" || echo "(blank)" } # Subfolders under / in a bucket ("(empty)" if none, or the S3 error code). From d6e2ebed34e92c5e9e720aa6ce97560f9f4fb83f Mon Sep 17 00:00:00 2001 From: jkrick Date: Thu, 29 Oct 2026 17:36:22 -0700 Subject: [PATCH 4/4] Run the whole SDT notebook in the SPHEREx release skill's test The old test ran selected cells and stood in a 2MASS star for secs. 9-11, which hid sec. 6 picking a D5 image instead of D4 on DR1. The script now executes every cell on a copy against the pre-release host (SIA, IBE in place of S3, that host's Firefly viewer) and prints a summary of what to check. Also record the DR1 gotchas: detector selection by name, the hard-coded selected_ids source, Firefly and MOST on pre-release hosts. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/spherex-data-release/SKILL.md | 23 +- .../references/decisions.md | 24 +- .../references/touchpoints.md | 12 +- .../scripts/run_sdt_release_test.py | 226 +++++++----------- 4 files changed, 131 insertions(+), 154 deletions(-) diff --git a/.claude/skills/spherex-data-release/SKILL.md b/.claude/skills/spherex-data-release/SKILL.md index 6395cddb..0d921bb3 100644 --- a/.claude/skills/spherex-data-release/SKILL.md +++ b/.claude/skills/spherex-data-release/SKILL.md @@ -22,8 +22,8 @@ those answers right away. asking the user anything; most questions were already answered once. - `scripts/check_release.sh ` -- what is live for release `` (e.g. `dr2`) on the pre-release and production hosts, S3 buckets and docs. -- `scripts/run_sdt_release_test.py` -- runs the Source Discovery Tool's own notebook cells against - pre-release data, with stand-ins for the parts that are not live yet. +- `scripts/run_sdt_release_test.py` -- executes the whole Source Discovery Tool notebook against + pre-release data and prints a summary of what to check. ## Workflow @@ -90,18 +90,23 @@ the plan what each commit did. ### 5. Test the Source Discovery Tool -The SDT needs SIA results and S3 reads that do not exist before release. -`scripts/run_sdt_release_test.py` runs the notebook's own cells with stand-ins: pre-release TAP rows -instead of SIA (with a `cloud_access` value built from the URI), pre-release IBE instead of S3, and -one 2MASS star instead of the SExtractor/interactive selection. Run it from the repo root: +Run the whole notebook, not selected cells. A DR1 test that ran only some cells and stood in a +2MASS star for secs. 9-11 passed while sec. 6 was silently picking a D5 image instead of D4 +(see decisions.md). `scripts/run_sdt_release_test.py` executes every cell on a copy, with the +pre-release host's SIA, its IBE mirror in place of S3, and its Firefly viewer. It needs the +release's SIA collection on that host and SExtractor's `sex` command (`brew install sextractor` +is enough; the notebook's conda env is not needed). Run it from the repo root: ```bash .venv/bin/python .claude/skills/spherex-data-release/scripts/run_sdt_release_test.py \ - tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md --release dr2 + tutorials/spherex/spherex_source_discovery/spherex_source_discovery_tool_demo.md --release dr2 \ + --out /sdt_run ``` -Report the number of spectral points, how many are flagged, and the flux near 2.16 um against -the 2MASS K estimate. For DR1: 281 points, 40 flagged with the final `bad_flags`, about 17.9 mJy. +Check its summary: sec. 6 must pick D2 and D4, the hard-coded `selected_ids` source must exist, +and its spectrum should look like a real source. Ask the user to look at the two Firefly tabs. +For DR1: 33 notebook code cells, 0 errors; 79 SExtractor sources, #65 was 40th by flux; 276 +points, 0.75-4.99 um, median S/N about 11, 50 points with a `bad_flags` pixel in the aperture. ### 6. Release day diff --git a/.claude/skills/spherex-data-release/references/decisions.md b/.claude/skills/spherex-data-release/references/decisions.md index abf040b6..49641224 100644 --- a/.claude/skills/spherex-data-release/references/decisions.md +++ b/.claude/skills/spherex-data-release/references/decisions.md @@ -42,6 +42,20 @@ collections, so check irsadev collections with a TAP query on `ivoa.obscore` instead. irsadev SIA `access_url`s point at a dev IBE host. IRSA may load only part of the sky, so confirm the notebook positions are covered. +- Select detectors by name, never by position. The SDT once took D4 as + `results.group_by('energy_bandpassname').groups[3]`, which is D4 only if all six detectors have + images at the position. DR1 at the demo position had D1, D2, D4 and D5, so it silently used D5 + with every label still saying D4. Fixed for DR1 by matching `groups.keys` to `'SPHEREx-D4'`. +- The SDT's sec. 12.1 measures a hard-coded SExtractor source number (`selected_ids = [65]`). The + numbering changes whenever the images change, so check the number on every release, including + release day. For DR1 it was left as 65 (a real source, mid-brightness) with a release-day check. +- Production Firefly (irsa.ipac.caltech.edu/irsaviewer) can't load images from pre-release hosts: + it shows "failed". Use the same host's viewer (`https://irsatest.ipac.caltech.edu/irsaviewer`) + when testing, and check production Firefly on release day. +- MOST on the pre-release hosts doesn't index the new release, so `MOST_queries.md` can't be + tested end to end before release. Its SPHEREx cutout cell can be: take the exposures production + MOST returns, find the new-release file with the same observation ID + (`level2___`) on the pre-release IBE, and run the cell on it. - Expect overlapping cleanup PRs. Merge them into the release branch before opening the release PR, or the same changes show up twice. @@ -54,9 +68,13 @@ `grep -rn "irsatest\|irsadev\|IRSA_HOST" tutorials/` finds nothing. 3. Put in the new DOI (SDT acknowledgements). 4. Bump `**Updated:**` in every notebook that changed. -5. Run each notebook against production with `jupytext --to notebook --execute` (SDT needs its - conda env, `conda-spherex_sdt.yml`). -6. Re-run `techniques-and-tools/MOST_queries.md` once the release is in MOST. +5. Run each notebook against production with `jupytext --to notebook --execute` (SDT needs + SExtractor: its conda env, `conda-spherex_sdt.yml`, or `brew install sextractor`). Check the + Firefly tabs show images, and that the SDT's hard-coded `selected_ids` still gives a real + spectrum. +6. `techniques-and-tools/MOST_queries.md` belongs to other authors and isn't in the release PR. + For DR1 a separate issue asked them to re-run it once MOST serves the release and check which + release MOST returns. 7. Recheck the supplement's flag table number. 8. Mark the draft PR ready for review (it closes the release issue). 9. Update this skill with what was learned. diff --git a/.claude/skills/spherex-data-release/references/touchpoints.md b/.claude/skills/spherex-data-release/references/touchpoints.md index 0701eb9d..a71e8645 100644 --- a/.claude/skills/spherex-data-release/references/touchpoints.md +++ b/.claude/skills/spherex-data-release/references/touchpoints.md @@ -46,10 +46,15 @@ quoted text instead. Paths are relative to `tutorials/spherex/`. `solid_angle_pixel_map/`, not `sapm/`). If only the SIA name changes, set `sia_collection` directly. Remove any "valid as of " note that no longer holds. - "Note that we have used the collection `spherex_dr1`" sentence after the SIA query. +- Sec. 6 picks the D2 and D4 images by `energy_bandpassname` (D4 through `groups.keys`, since DR1). + Check that the new release's search still returns both detectors at the demo position. - Sec. 7 `get_s3_fpath(cloud_access)`: needs `cloud_access` filled in SIA results. - Sec. 8 masks: D2 mask from `flags_data2`, D4 mask from `flags_data4` (DR1 fixed a typo here). - Sec. 12.1 TAP query: `IRSA_HOST` test block (uses `pyvo.dal.TAPService(f"{IRSA_HOST}/TAP")`), release filter, cutout URL prefix. +- Sec. 12.1 `selected_ids = [65]`: a SExtractor source number from sec. 10, so it changes when the + images do. Check it still names a real source. +- Secs. 6 and 11.2 use `FireflyClient.make_lab_client()`, which works only inside JupyterLab. - Sec. 12.2 `bad_flags`: the supplement's CONTAINS_BAD_PIXEL list (see decisions.md). Any listed flag in the aperture drops the point from the plotted spectrum (`bokeh_viz.plot_spectrum`). - Acknowledgements: DOI sentence. Cite only the DOIs of releases the notebook actually loads. @@ -68,8 +73,11 @@ quoted text instead. Paths are relative to `tutorials/spherex/`. ## techniques-and-tools/MOST_queries.md -- No release-specific code (`catalog="spherex"`). Re-run once the release is queryable in MOST to - refresh its outputs. +- Not a SPHEREx tutorial (other authors) and not changed for DR1. Sec. 5 queries MOST with + `catalog="spherex"` (Vesta, Aug 2025); `plot_path()` uses only MOST's output columns; the cutout + cell reads `hdul[1]` and `PC1_1`/`PC1_2`. No release-specific code. Before DR1, production MOST + returned QR2 images; whether it returns the new release, the old one, or both is only known + after release (see decisions.md for the pre-release check). ## Repo files diff --git a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py index 3f305657..8acc4beb 100644 --- a/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py +++ b/.claude/skills/spherex-data-release/scripts/run_sdt_release_test.py @@ -1,159 +1,105 @@ -"""Run the Source Discovery Tool notebook's own sec. 5, 7, 8 and 12 cells against a SPHEREx -release that is not yet in SIA or public S3. +"""Execute the whole Source Discovery Tool notebook against a SPHEREx release that is not public yet. -Usage (from the repo root, IRSA network or VPN): +Usage (from the repo root, IRSA network or VPN; needs the `sex` command from SExtractor -- +`brew install sextractor` works, no conda env needed): python run_sdt_release_test.py \ - --release dr2 [--host https://irsatest.ipac.caltech.edu] \ - [--sia-host https://irsadev.ipac.caltech.edu] + --release dr2 [--host https://irsatest.ipac.caltech.edu] [--out sdt_dr2_test_run] -Stand-ins: -- sec. 6 SIA results -> TAP rows on --host, with a `cloud_access` JSON built from the uri. - With --sia-host, sec. 6 instead runs the notebook's own SIA cells against that host (use this - once IRSA has loaded the release's SIA collections there). +Runs every cell (secs. 4-12, including SExtractor, bokeh and Firefly) on a copy of the notebook's +folder in --out; the tracked notebook is not changed. Test-only changes to the copy: +- SIA -> --host. IRSA must have loaded the release's SIA collection there (check_release.sh). - s3://nasa-irsa-spherex//... reads -> the IBE mirror of the same path on --host -- secs. 10-11 (SExtractor + interactive selection) -> one 2MASS star near the field center -Secs. 9-11 (SExtractor, Firefly, bokeh) are not exercised. + (public S3 is empty before release). +- FireflyClient.make_lab_client() -> make_client() on --host's irsaviewer. A headless run has no + JupyterLab, and production Firefly can't load pre-release URLs. Browser tabs open; look at them. +At the end it prints a summary: which detectors sec. 6 picked, the SExtractor catalog, the +hard-coded sec. 12.1 source, and its spectrum. """ import argparse import os +import shutil import sys -import json -import traceback import jupytext -import astropy.io.fits as fits_mod +import nbformat +from nbclient import NotebookClient _ap = argparse.ArgumentParser() _ap.add_argument("notebook") _ap.add_argument("--release", required=True, help="e.g. dr2") _ap.add_argument("--host", default="https://irsatest.ipac.caltech.edu") -_ap.add_argument("--sia-host", help="run sec. 6 with real SIA from this host") +_ap.add_argument("--out", help="run folder (default: sdt__test_run in the current folder)") ARGS = _ap.parse_args() REL, HOST = ARGS.release, ARGS.host.rstrip("/") - -NB = ARGS.notebook -SDT_DIR = os.path.dirname(os.path.abspath(NB)) -sys.path.insert(0, SDT_DIR) - -S3_PREFIX = f"s3://nasa-irsa-spherex/{REL}/" -IBE_PREFIX = f"{HOST}/ibe/data/spherex/{REL}/" -_real_open = fits_mod.open - - -def open_via_ibe(name, *args, **kwargs): - if isinstance(name, str) and name.startswith(S3_PREFIX): +OUT = os.path.abspath(ARGS.out or f"sdt_{REL}_test_run") +NB_NAME = os.path.basename(ARGS.notebook) + +if not shutil.which("sex"): + sys.exit("SExtractor's `sex` command is not on PATH (try `brew install sextractor`).") + +OVERRIDE = f''' +# TEST HARNESS ONLY (run_sdt_release_test.py) +Irsa.sia_url = "{HOST}/SIA" +_S3, _IBE = "s3://nasa-irsa-spherex/{REL}/", "{HOST}/ibe/data/spherex/{REL}/" +_real_open = fits.open +def _open_via_ibe(name, *args, **kwargs): + if isinstance(name, str) and name.startswith(_S3): kwargs.pop("fsspec_kwargs", None) - name = IBE_PREFIX + name[len(S3_PREFIX):] + name = _IBE + name[len(_S3):] return _real_open(name, *args, **kwargs) - - -fits_mod.open = open_via_ibe - -cells = [c["source"] for c in jupytext.read(NB).cells if c["cell_type"] == "code"] - - -def cell(marker): - hits = [c for c in cells if marker in c] - assert len(hits) == 1, (marker, len(hits)) - return hits[0] - - -ns = {"HOST": HOST, "REL": REL} - - -def run(label, src): - print(f"\n===== {label} =====", flush=True) - try: - exec(src, ns) - return True - except Exception: - traceback.print_exc() - return False - - -run("imports", cell("import concurrent.futures")) -run("sec 5 params", cell('data_release = "')) -print("sapm_s3[1] =", ns["sapm_s3"][1]) - -if ARGS.sia_host: - # ---- sec. 6 for real, against the pre-release SIA host - ns["Irsa"].sia_url = ARGS.sia_host.rstrip("/") + "/SIA" - run("sec 6 coord", cell("search_radius = 50 * u.arcsec")) - run("sec 6 SIA query", cell("results = Irsa.query_sia(")) - run("sec 6 pick D2/D4", cell("d2_result = results[")) - print(len(ns["results"]), "SIA rows; D2 cloud_access:", str(ns["d2_result"]["cloud_access"])[:120]) - -# ---- sec. 6 stand-in: first D2 and D4 images of the release at the SDT position, from TAP -else: - run("sec 6 stand-in", ''' -coord = SkyCoord(150.7817877, 3.3502204, unit='deg') -_svc = pyvo.dal.TAPService(HOST + "/TAP") -_q = """SELECT a.uri, p.energy_bandpassname, p.time_bounds_lower -FROM spherex.artifact a JOIN spherex.plane p ON a.planeid = p.planeid -WHERE 1 = CONTAINS(POINT('ICRS', 150.7817877, 3.3502204), p.poly) - AND a.uri LIKE 'ibe/data/spherex/%s/%%' -ORDER BY p.time_bounds_lower""" % REL -_t = _svc.search(_q).to_table() -print(len(_t), REL, "images at SDT position;", sorted(set(_t["energy_bandpassname"]))) -def _fake_row(det): - r = _t[_t["energy_bandpassname"] == det][0] - key = r["uri"][len("ibe/data/spherex/"):] - return {"cloud_access": json.dumps({"aws": {"bucket_name": "nasa-irsa-spherex", "key": key, - "region": "us-east-1"}})} -d2_result = _fake_row("SPHEREx-D2") -d4_result = _fake_row("SPHEREx-D4") -''') - -ok7 = run("sec 7 get_s3_fpath", cell("def get_s3_fpath")) and \ - run("sec 7 load images", cell("s3_fpath_d2 = get_s3_fpath")) -if ok7: - print("D2 path:", ns["s3_fpath_d2"]) -ok8 = ok7 and all([ - run("sec 8 SAPM", cell("s3_fpath_sapm2 = sapm_s3[1]")), - run("sec 8 units", cell('print(f"Spectral image BUNIT')), - run("sec 8 background", cell("bkg2 = sep.Background")), - run("sec 8 masks", cell("mask2 = np.zeros")), - run("sec 8 reproject", cell("img_data2_reproj, footprint = reproject_exact")), -]) -if ok8: - import numpy as np - print("D2 BUNIT now:", ns["img_hdr2"]["BUNIT"], "| reprojected finite px:", - int(np.isfinite(ns["img_data2_reproj"]).sum()), - "| good-pixel fraction D2 mask:", float(np.nanmean(ns["mask2"] == 1))) - -# ---- secs. 10-11 stand-in: one 2MASS star near the field center -run("matched_stars stand-in", ''' -import pandas as pd -# 2MASS from --host's TAP (production IRSA TAP can return 502s) -import pyvo as _pyvo -_stars = _pyvo.dal.TAPService(HOST + "/TAP").run_sync( - f"SELECT ra, dec, k_m FROM fp_psc WHERE CONTAINS(POINT('ICRS', ra, dec), " - f"CIRCLE('ICRS', {coord.ra.deg}, {coord.dec.deg}, {50/3600})) = 1").to_table() -_stars = _stars[(_stars["k_m"] > 11) & (_stars["k_m"] < 14)] -_stars.sort("k_m") -matched_stars = pd.DataFrame({"NUMBER": [65], "ra": [float(_stars["ra"][0])], "dec": [float(_stars["dec"][0])]}) -print(matched_stars, "Kmag", _stars["k_m"][0]) -''') - -ok12 = all([ - run("sec 12.1 TAP query", cell("service = pyvo.dal.TAPService")), - run("sec 12.1 process_cutout def", cell("def process_cutout")), - run("sec 12.1 results_table", cell('results_table["cutout_index"]')), - run("sec 12.1 download cutouts", cell("Process cutouts in parallel")), -]) -os.makedirs("data", exist_ok=True) -if ok12: - run("sec 12.2 build_phot_table", cell("build_phot_table(results_table")) - run("sec 12.2 load pickle", cell("with open(phot_path, 'rb')")) - if run("sec 12.2 grab_star", cell("bad_flags = [")): - print("n points:", len(ns["wvl"]), "| wl range:", ns["wvl"].min(), ns["wvl"].max()) - -if "wvl" in ns: - import numpy as np - w, f, nb = np.asarray(ns["wvl"]), np.asarray(ns["flx"]), np.asarray(ns["n_bad"]) - k = np.abs(w - 2.16) < 0.1 - print("flux near 2.16 um (mJy, median):", np.nanmedian(f[k]), "| expected for Kmag", - "~", 666.7e3 * 10 ** (-11.946 / 2.5)) - print("points with >=1 flagged pixel in aperture:", int((nb > 0).sum()), "of", len(nb)) - entry = ns["all_stars_band_phot"]["obs_log"][0][0] - print("flags recorded:", sorted(entry["dqf_sum"].keys())) +fits.open = _open_via_ibe +''' + +SUMMARY = ''' +# TEST HARNESS ONLY: summary +print("=== SUMMARY ===") +print("sec. 6 images:", d2_result["energy_bandpassname"], "and", d4_result["energy_bandpassname"], + "(expect SPHEREx-D2 and SPHEREx-D4)") +_by_flux = extracted.copy() +_by_flux.sort("FLUX_AUTO", reverse=True) +_nums = list(_by_flux["NUMBER"]) +for _sid in selected_ids: + _rank = _nums.index(_sid) + 1 if _sid in _nums else None + print(f"SExtractor: {len(extracted)} sources; selected_ids {_sid} is", + f"#{_rank} by FLUX_AUTO" if _rank else "NOT in the catalog") +_w, _f, _v = np.asarray(wvl, float), np.asarray(flx, float), np.asarray(var, float) +_nb = np.asarray(n_bad, float) +print(f"spectrum: {len(_w)} points, {np.nanmin(_w):.2f}-{np.nanmax(_w):.2f} um,", + f"median S/N {np.nanmedian(_f / np.sqrt(_v)):.1f},", + f"{int((_nb > 0).sum())} points with a bad_flags pixel in the aperture") +''' + +if os.path.exists(OUT): + shutil.rmtree(OUT) +shutil.copytree(os.path.dirname(os.path.abspath(ARGS.notebook)), OUT) + +nb = jupytext.read(os.path.join(OUT, NB_NAME)) +code = [i for i, c in enumerate(nb.cells) if c.cell_type == "code"] +anchor = [i for i in code if "output_notebook(" in nb.cells[i].source] +assert len(anchor) == 1, "can't find the cell to insert the test overrides after" +n_ff = 0 +for i in code: + if "FireflyClient.make_lab_client()" in nb.cells[i].source: + n_ff += 1 + nb.cells[i].source = nb.cells[i].source.replace( + "FireflyClient.make_lab_client()", + f'FireflyClient.make_client(url="{HOST}/irsaviewer", channel_override="sdt_test_{n_ff}")') +print(f"Firefly cells redirected: {n_ff}") +nb.cells.insert(anchor[0] + 1, nbformat.v4.new_code_cell(OVERRIDE)) +nb.cells.append(nbformat.v4.new_code_cell(SUMMARY)) + +print(f"Executing {NB_NAME} in {OUT} (several minutes) ...", flush=True) +NotebookClient(nb, timeout=1800, kernel_name="python3", allow_errors=True, + resources={"metadata": {"path": OUT}}).execute() +nbformat.write(nb, os.path.join(OUT, NB_NAME.replace(".md", ".ipynb"))) + +n_err = 0 +for c in nb.cells: + for o in c.get("outputs", []) if c.cell_type == "code" else []: + if o.get("output_type") == "error": + n_err += 1 + print("=" * 70, "\n" + c.source[:300], "\n--> ERROR", o["ename"], o["evalue"][:500]) +summary = "".join(o.get("text", "") for o in nb.cells[-1].get("outputs", [])) +print("\n" + (summary or "(summary cell did not run)")) +print(f"{len(code)} notebook code cells, {n_err} errors. Executed copy: " + f"{os.path.join(OUT, NB_NAME.replace('.md', '.ipynb'))}")