From 0c49e0a65602552d71c87dfffe50a18d83172f10 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 15:40:21 -0400 Subject: [PATCH 1/6] tools/verify_release.py: hash a release directory; re-verify any host or mirror against it (chain step 8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hash: every .parquet/.json under a dir → release_hashes.json (bytes + sha256). check: HEAD size fast-fail then streamed sha256 per file against --base URL or --dir; --only/--skip-prefix (skipped is reported, never counted as verified); JSON --report; exit 0 only when every listed file matches. Explicit User-Agent (Cloudflare 403s Python-urllib). Tested: 16 local files verified; live vocab_labels_202608 verified against data.isamples.org; tampered sha256 and wrong size both FAIL with exit 1. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/verify_release.py | 192 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 tools/verify_release.py diff --git a/tools/verify_release.py b/tools/verify_release.py new file mode 100644 index 00000000..75355d08 --- /dev/null +++ b/tools/verify_release.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""verify_release.py — hash a release's data files, and re-verify any host against those hashes. + +Two modes: + + hash Walk a release directory and write a *release hash manifest*: every + .parquet/.json file (relative path, bytes, sha256). This is the + machine-readable statement of "these exact bytes are release X". + + python3 tools/verify_release.py hash --dir ~/Data/iSample/pqg_refining/202609/publish \ + --release-id isamples_202609 --out provenance/isamples_202609/release_hashes.json + + check Re-download (or re-read) every file named in that manifest from a host + (--base https://data.isamples.org) or a directory (--dir) and compare + size and sha256. Exit 0 only if every file matches. + + python3 tools/verify_release.py check --manifest provenance/isamples_202609/release_hashes.json \ + --base https://data.isamples.org + python3 tools/verify_release.py check --manifest ... --dir /path/to/mirror --only 'isamples_202609_h3_*' + + Files are streamed; nothing is kept on disk. A wrong size fails fast + (HEAD) before the body is fetched. --only takes a glob over the + relative path; --skip-prefix drops e.g. the 900-shard search index + when a quick check is wanted (say so in the report — a skipped file is + not a verified file). + +This is the reproducibility programme's step 8 (REPRODUCIBLE_PIPELINE_PLAN_2026-08-25.md): +a build's hashes (from the per-step manifests) are recorded once, and any copy — +R2, a mirror, a Zenodo deposit unpacked locally — can be checked against them. +The #334 release manifest (size/etag) stays the Explorer's boot-time cross-check; +this one is the byte-level truth. +""" +import argparse +import datetime +import fnmatch +import hashlib +import json +import os +import sys +import urllib.error +import urllib.request + +CHUNK = 1 << 20 +EXTS = (".parquet", ".json") +# data.isamples.org sits behind Cloudflare, which answers urllib's default +# "Python-urllib" agent with 403; identify ourselves instead. +UA = "isamples-verify-release/1 (+https://github.com/isamplesorg/isamplesorg.github.io)" + + +def _req(url, method="GET"): + return urllib.request.Request(url, method=method, headers={"User-Agent": UA}) + + +def sha256_path(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(CHUNK), b""): + h.update(chunk) + return h.hexdigest() + + +def sha256_url(url, expected_bytes=None, timeout=60): + """Stream a URL, returning (bytes_read, sha256). HEAD first to fail fast on size.""" + with urllib.request.urlopen(_req(url, "HEAD"), timeout=timeout) as r: + cl = r.headers.get("Content-Length") + if expected_bytes is not None and cl is not None and int(cl) != expected_bytes: + return int(cl), None # size mismatch: don't bother downloading + h = hashlib.sha256() + n = 0 + with urllib.request.urlopen(_req(url), timeout=timeout) as r: + for chunk in iter(lambda: r.read(CHUNK), b""): + h.update(chunk) + n += len(chunk) + return n, h.hexdigest() + + +def cmd_hash(args): + root = os.path.abspath(args.dir) + files = {} + for dirpath, _, names in os.walk(root): + for name in sorted(names): + if not name.endswith(EXTS) or name.endswith(".manifest.json"): + continue + full = os.path.join(dirpath, name) + rel = os.path.relpath(full, root).replace(os.sep, "/") + if args.only and not fnmatch.fnmatch(rel, args.only): + continue + files[rel] = {"bytes": os.path.getsize(full), "sha256": sha256_path(full)} + print(f" {files[rel]['sha256'][:12]} {files[rel]['bytes']:>12,} {rel}") + if not files: + print("ERROR: no files found", file=sys.stderr) + return 2 + doc = { + "schema": "release_hashes/1", + "release_id": args.release_id, + "generated_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + "source_dir": root.replace(os.path.expanduser("~"), "~"), + "file_count": len(files), + "total_bytes": sum(f["bytes"] for f in files.values()), + "files": dict(sorted(files.items())), + } + if args.out: + os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True) + with open(args.out, "w") as fh: + json.dump(doc, fh, indent=1) + print(f"wrote {args.out}: {len(files)} files, {doc['total_bytes']/1e6:.1f} MB") + else: + json.dump(doc, sys.stdout, indent=1) + return 0 + + +def cmd_check(args): + with open(args.manifest) as fh: + doc = json.load(fh) + files = doc["files"] + if bool(args.base) == bool(args.dir): + print("ERROR: give exactly one of --base or --dir", file=sys.stderr) + return 2 + base = args.base.rstrip("/") if args.base else None + ok = mismatch = missing = skipped = 0 + rows = [] + for rel, exp in files.items(): + if args.only and not fnmatch.fnmatch(rel, args.only): + skipped += 1 + continue + if args.skip_prefix and rel.startswith(args.skip_prefix): + skipped += 1 + continue + try: + if base: + n, digest = sha256_url(f"{base}/{rel}", exp["bytes"], timeout=args.timeout) + else: + path = os.path.join(args.dir, rel) + if not os.path.exists(path): + raise FileNotFoundError(path) + n = os.path.getsize(path) + digest = sha256_path(path) if n == exp["bytes"] else None + except (urllib.error.HTTPError, urllib.error.URLError, FileNotFoundError) as e: + missing += 1 + rows.append(("MISSING", rel, str(e)[:80])) + print(f" MISSING {rel} ({str(e)[:60]})") + continue + if n != exp["bytes"]: + mismatch += 1 + rows.append(("SIZE", rel, f"{n} != {exp['bytes']}")) + print(f" SIZE {rel} {n:,} != {exp['bytes']:,}") + elif digest != exp["sha256"]: + mismatch += 1 + rows.append(("SHA256", rel, f"{digest[:12]} != {exp['sha256'][:12]}")) + print(f" SHA256 {rel} {digest[:12]}… != {exp['sha256'][:12]}…") + else: + ok += 1 + if args.verbose: + print(f" ok {rel}") + target = base or os.path.abspath(args.dir) + verdict = "VERIFIED" if (mismatch == 0 and missing == 0 and ok > 0) else "FAILED" + print(f"\n{verdict}: {doc.get('release_id')} on {target} — {ok} ok, {mismatch} mismatched, {missing} missing, " + f"{skipped} skipped (of {len(files)} listed)" + ("" if not skipped else " [skipped files are NOT verified]")) + if args.report: + with open(args.report, "w") as fh: + json.dump({"release_id": doc.get("release_id"), "target": target, "verdict": verdict, + "checked_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + "ok": ok, "mismatched": mismatch, "missing": missing, "skipped": skipped, + "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in rows]}, fh, indent=1) + return 0 if verdict == "VERIFIED" else 1 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[1], formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + h = sub.add_parser("hash", help="write a release hash manifest from a directory") + h.add_argument("--dir", required=True) + h.add_argument("--release-id", required=True) + h.add_argument("--out") + h.add_argument("--only", help="glob over the relative path") + h.set_defaults(fn=cmd_hash) + c = sub.add_parser("check", help="verify a host or directory against a release hash manifest") + c.add_argument("--manifest", required=True) + c.add_argument("--base", help="e.g. https://data.isamples.org") + c.add_argument("--dir", help="local mirror directory") + c.add_argument("--only", help="glob over the relative path") + c.add_argument("--skip-prefix", help="skip files under this relative prefix (reported as skipped)") + c.add_argument("--timeout", type=int, default=120) + c.add_argument("--report", help="write a JSON report") + c.add_argument("-v", "--verbose", action="store_true") + c.set_defaults(fn=cmd_check) + args = ap.parse_args() + return args.fn(args) + + +if __name__ == "__main__": + sys.exit(main()) From d59a289ad4fd59fc4af2958e5dada7c89348254e Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 15:46:35 -0400 Subject: [PATCH 2/6] verify_release.py: VERIFIED only when nothing was skipped (PARTIAL/3 otherwise); refuse redirects; URL-encode + validate paths; contain --dir; strict GET framing; complete hash mode (Codex round 1) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/verify_release.py | 229 +++++++++++++++++++++++++++------------- 1 file changed, 158 insertions(+), 71 deletions(-) diff --git a/tools/verify_release.py b/tools/verify_release.py index 75355d08..a3fe5f1a 100644 --- a/tools/verify_release.py +++ b/tools/verify_release.py @@ -4,94 +4,139 @@ Two modes: hash Walk a release directory and write a *release hash manifest*: every - .parquet/.json file (relative path, bytes, sha256). This is the - machine-readable statement of "these exact bytes are release X". + .parquet/.json file (relative path, bytes, sha256). Always complete — + there is deliberately no filter, so a manifest can never look + authoritative while omitting files. python3 tools/verify_release.py hash --dir ~/Data/iSample/pqg_refining/202609/publish \ --release-id isamples_202609 --out provenance/isamples_202609/release_hashes.json check Re-download (or re-read) every file named in that manifest from a host (--base https://data.isamples.org) or a directory (--dir) and compare - size and sha256. Exit 0 only if every file matches. + size and sha256. python3 tools/verify_release.py check --manifest provenance/isamples_202609/release_hashes.json \ --base https://data.isamples.org python3 tools/verify_release.py check --manifest ... --dir /path/to/mirror --only 'isamples_202609_h3_*' - Files are streamed; nothing is kept on disk. A wrong size fails fast - (HEAD) before the body is fetched. --only takes a glob over the - relative path; --skip-prefix drops e.g. the 900-shard search index - when a quick check is wanted (say so in the report — a skipped file is - not a verified file). + Verdicts and exit codes: + VERIFIED (0) every listed file was checked and matches + PARTIAL (3) every *checked* file matches but --only/--skip-prefix left + some unchecked; exit 0 only with --allow-partial + FAILED (1) a mismatch, a missing file, or an operational error + Files are streamed, never stored. Redirects are refused (a "mirror" + that redirects to the origin is not a copy). Bodies are requested + unencoded (Accept-Encoding: identity) and any Content-Encoding fails + the file. A wrong HEAD size fails fast before the body is fetched. This is the reproducibility programme's step 8 (REPRODUCIBLE_PIPELINE_PLAN_2026-08-25.md): -a build's hashes (from the per-step manifests) are recorded once, and any copy — -R2, a mirror, a Zenodo deposit unpacked locally — can be checked against them. -The #334 release manifest (size/etag) stays the Explorer's boot-time cross-check; -this one is the byte-level truth. +a build's hashes are recorded once, and any copy — R2, a mirror, a Zenodo deposit +unpacked locally — can be checked against them. The #334 release manifest +(size/etag) stays the Explorer's boot-time cross-check; this one is the +byte-level truth. """ import argparse import datetime import fnmatch import hashlib +import http.client import json import os +import re import sys import urllib.error +import urllib.parse import urllib.request CHUNK = 1 << 20 EXTS = (".parquet", ".json") +SCHEMA = "release_hashes/1" +# Relative paths a manifest may contain: plain segments, '/' separators, no '..', +# no absolute paths, no backslashes, no URL-significant characters. +PATH_RE = re.compile(r"^(?!/)(?!.*(^|/)\.\.(/|$))[A-Za-z0-9._\-/]+$") # data.isamples.org sits behind Cloudflare, which answers urllib's default # "Python-urllib" agent with 403; identify ourselves instead. UA = "isamples-verify-release/1 (+https://github.com/isamplesorg/isamplesorg.github.io)" -def _req(url, method="GET"): - return urllib.request.Request(url, method=method, headers={"User-Agent": UA}) +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.HTTPError(req.full_url, code, f"redirect to {newurl} refused (not a copy)", headers, fp) -def sha256_path(path): - h = hashlib.sha256() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(CHUNK), b""): - h.update(chunk) - return h.hexdigest() +_OPENER = urllib.request.build_opener(_NoRedirect) -def sha256_url(url, expected_bytes=None, timeout=60): - """Stream a URL, returning (bytes_read, sha256). HEAD first to fail fast on size.""" - with urllib.request.urlopen(_req(url, "HEAD"), timeout=timeout) as r: - cl = r.headers.get("Content-Length") - if expected_bytes is not None and cl is not None and int(cl) != expected_bytes: - return int(cl), None # size mismatch: don't bother downloading +def _open(url, method="GET", timeout=120): + req = urllib.request.Request(url, method=method, + headers={"User-Agent": UA, "Accept-Encoding": "identity"}) + return _OPENER.open(req, timeout=timeout) + + +def hash_and_count(fobj): + """Return (bytes_read, sha256) over one read pass, so size and digest agree.""" h = hashlib.sha256() n = 0 - with urllib.request.urlopen(_req(url), timeout=timeout) as r: - for chunk in iter(lambda: r.read(CHUNK), b""): - h.update(chunk) - n += len(chunk) + for chunk in iter(lambda: fobj.read(CHUNK), b""): + h.update(chunk) + n += len(chunk) return n, h.hexdigest() +def fetch_hash(url, expected_bytes, timeout): + """Stream `url`; return (bytes_read, sha256 | None, note). Fails fast on a + HEAD size mismatch; verifies GET status/framing; a short body is a failure.""" + try: + with _open(url, "HEAD", timeout) as r: + cl = r.headers.get("Content-Length") + if cl is not None and int(cl) != expected_bytes: + return int(cl), None, "HEAD size" + except urllib.error.HTTPError as e: + if e.code not in (405, 501): # HEAD unsupported: fall through to GET + raise + with _open(url, "GET", timeout) as r: + if r.status != 200: + raise urllib.error.HTTPError(url, r.status, "non-200 GET", r.headers, None) + enc = r.headers.get("Content-Encoding") + if enc and enc.lower() != "identity": + return 0, None, f"Content-Encoding {enc}" + cl = r.headers.get("Content-Length") + if cl is not None and int(cl) != expected_bytes: + return int(cl), None, "GET size" + n, digest = hash_and_count(r) + return n, digest, "" + + def cmd_hash(args): - root = os.path.abspath(args.dir) + root = os.path.realpath(args.dir) files = {} - for dirpath, _, names in os.walk(root): + skipped_links = [] + for dirpath, dirnames, names in os.walk(root, followlinks=False): + dirnames.sort() for name in sorted(names): if not name.endswith(EXTS) or name.endswith(".manifest.json"): continue full = os.path.join(dirpath, name) rel = os.path.relpath(full, root).replace(os.sep, "/") - if args.only and not fnmatch.fnmatch(rel, args.only): + if os.path.islink(full): + skipped_links.append(rel) # a symlink is not release content continue - files[rel] = {"bytes": os.path.getsize(full), "sha256": sha256_path(full)} - print(f" {files[rel]['sha256'][:12]} {files[rel]['bytes']:>12,} {rel}") + if not PATH_RE.match(rel): + print(f"ERROR: path not representable in a manifest: {rel}", file=sys.stderr) + return 2 + with open(full, "rb") as f: + n, digest = hash_and_count(f) + files[rel] = {"bytes": n, "sha256": digest} + print(f" {digest[:12]} {n:>12,} {rel}") + if skipped_links: + print(f"ERROR: {len(skipped_links)} symlinked file(s) in the release dir (not hashed): " + + ", ".join(skipped_links[:5]), file=sys.stderr) + return 2 if not files: print("ERROR: no files found", file=sys.stderr) return 2 doc = { - "schema": "release_hashes/1", + "schema": SCHEMA, "release_id": args.release_id, "generated_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), "source_dir": root.replace(os.path.expanduser("~"), "~"), @@ -109,77 +154,119 @@ def cmd_hash(args): return 0 -def cmd_check(args): - with open(args.manifest) as fh: +def load_manifest(path): + with open(path) as fh: doc = json.load(fh) - files = doc["files"] + if doc.get("schema") != SCHEMA: + raise ValueError(f"unexpected manifest schema {doc.get('schema')!r} (want {SCHEMA})") + files = doc.get("files") + if not isinstance(files, dict) or not files: + raise ValueError("manifest has no files") + if doc.get("file_count") != len(files): + raise ValueError(f"file_count {doc.get('file_count')} != {len(files)} entries") + total = 0 + for rel, e in files.items(): + if not PATH_RE.match(rel): + raise ValueError(f"bad path in manifest: {rel!r}") + if not (isinstance(e.get("bytes"), int) and e["bytes"] >= 0): + raise ValueError(f"bad bytes for {rel}") + if not (isinstance(e.get("sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", e["sha256"])): + raise ValueError(f"bad sha256 for {rel}") + total += e["bytes"] + if doc.get("total_bytes") != total: + raise ValueError(f"total_bytes {doc.get('total_bytes')} != sum {total}") + return doc + + +def cmd_check(args): if bool(args.base) == bool(args.dir): print("ERROR: give exactly one of --base or --dir", file=sys.stderr) return 2 + try: + doc = load_manifest(args.manifest) + except (OSError, ValueError, json.JSONDecodeError) as e: + print(f"ERROR: manifest rejected: {e}", file=sys.stderr) + return 2 + files = doc["files"] base = args.base.rstrip("/") if args.base else None - ok = mismatch = missing = skipped = 0 - rows = [] + root = os.path.realpath(args.dir) if args.dir else None + ok, problems, skipped = 0, [], [] for rel, exp in files.items(): - if args.only and not fnmatch.fnmatch(rel, args.only): - skipped += 1 - continue - if args.skip_prefix and rel.startswith(args.skip_prefix): - skipped += 1 + if (args.only and not fnmatch.fnmatch(rel, args.only)) or (args.skip_prefix and rel.startswith(args.skip_prefix)): + skipped.append(rel) continue try: if base: - n, digest = sha256_url(f"{base}/{rel}", exp["bytes"], timeout=args.timeout) + url = base + "/" + "/".join(urllib.parse.quote(seg, safe="") for seg in rel.split("/")) + n, digest, note = fetch_hash(url, exp["bytes"], args.timeout) else: - path = os.path.join(args.dir, rel) - if not os.path.exists(path): + path = os.path.realpath(os.path.join(root, rel)) + if os.path.commonpath([root, path]) != root: + raise ValueError("path escapes --dir") + if os.path.islink(os.path.join(root, rel)): + raise ValueError("symlink, not a copy") + if not os.path.isfile(path): raise FileNotFoundError(path) n = os.path.getsize(path) - digest = sha256_path(path) if n == exp["bytes"] else None + note = "size" + digest = None + if n == exp["bytes"]: + with open(path, "rb") as f: + n, digest = hash_and_count(f) + note = "" except (urllib.error.HTTPError, urllib.error.URLError, FileNotFoundError) as e: - missing += 1 - rows.append(("MISSING", rel, str(e)[:80])) - print(f" MISSING {rel} ({str(e)[:60]})") + problems.append(("MISSING", rel, str(e)[:100])) + print(f" MISSING {rel} ({str(e)[:70]})") continue - if n != exp["bytes"]: - mismatch += 1 - rows.append(("SIZE", rel, f"{n} != {exp['bytes']}")) - print(f" SIZE {rel} {n:,} != {exp['bytes']:,}") + except (OSError, ValueError, http.client.HTTPException, TimeoutError) as e: + problems.append(("ERROR", rel, f"{type(e).__name__}: {str(e)[:100]}")) + print(f" ERROR {rel} ({type(e).__name__}: {str(e)[:60]})") + continue + if digest is None or n != exp["bytes"]: + problems.append(("SIZE", rel, f"{n} != {exp['bytes']} ({note})")) + print(f" SIZE {rel} {n:,} != {exp['bytes']:,} ({note})") elif digest != exp["sha256"]: - mismatch += 1 - rows.append(("SHA256", rel, f"{digest[:12]} != {exp['sha256'][:12]}")) + problems.append(("SHA256", rel, f"{digest[:12]} != {exp['sha256'][:12]}")) print(f" SHA256 {rel} {digest[:12]}… != {exp['sha256'][:12]}…") else: ok += 1 if args.verbose: print(f" ok {rel}") - target = base or os.path.abspath(args.dir) - verdict = "VERIFIED" if (mismatch == 0 and missing == 0 and ok > 0) else "FAILED" - print(f"\n{verdict}: {doc.get('release_id')} on {target} — {ok} ok, {mismatch} mismatched, {missing} missing, " - f"{skipped} skipped (of {len(files)} listed)" + ("" if not skipped else " [skipped files are NOT verified]")) + target = base or root + if problems or ok == 0: + verdict, code = "FAILED", 1 + elif skipped: + verdict, code = "PARTIAL", (0 if args.allow_partial else 3) + else: + verdict, code = "VERIFIED", 0 + print(f"\n{verdict}: {doc['release_id']} on {target} — {ok} ok, {len(problems)} problem(s), " + f"{len(skipped)} skipped (of {len(files)} listed)" + + ("" if not skipped else " [skipped files were NOT checked]")) if args.report: with open(args.report, "w") as fh: - json.dump({"release_id": doc.get("release_id"), "target": target, "verdict": verdict, + json.dump({"release_id": doc["release_id"], "target": target, "verdict": verdict, "exit_code": code, "checked_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), - "ok": ok, "mismatched": mismatch, "missing": missing, "skipped": skipped, - "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in rows]}, fh, indent=1) - return 0 if verdict == "VERIFIED" else 1 + "manifest": os.path.abspath(args.manifest), "only": args.only, "skip_prefix": args.skip_prefix, + "ok": ok, "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in problems], + "skipped": skipped}, fh, indent=1) + return code def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[1], formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) - h = sub.add_parser("hash", help="write a release hash manifest from a directory") + h = sub.add_parser("hash", help="write a complete release hash manifest from a directory") h.add_argument("--dir", required=True) h.add_argument("--release-id", required=True) h.add_argument("--out") - h.add_argument("--only", help="glob over the relative path") h.set_defaults(fn=cmd_hash) c = sub.add_parser("check", help="verify a host or directory against a release hash manifest") c.add_argument("--manifest", required=True) c.add_argument("--base", help="e.g. https://data.isamples.org") c.add_argument("--dir", help="local mirror directory") - c.add_argument("--only", help="glob over the relative path") - c.add_argument("--skip-prefix", help="skip files under this relative prefix (reported as skipped)") + c.add_argument("--only", help="glob over the relative path (verdict becomes PARTIAL)") + c.add_argument("--skip-prefix", help="skip files under this relative prefix (verdict becomes PARTIAL)") + c.add_argument("--allow-partial", action="store_true", help="exit 0 on PARTIAL") c.add_argument("--timeout", type=int, default=120) c.add_argument("--report", help="write a JSON report") c.add_argument("-v", "--verbose", action="store_true") From bc3c70bb281062f7c16b161852e30cf80882ca7d Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 15:54:35 -0400 Subject: [PATCH 3/6] verify_release.py: report/out containment, base URL validation, complete walks, strict manifest typing, canonical paths (Codex round 2) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/verify_release.py | 111 ++++++++++++++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 22 deletions(-) diff --git a/tools/verify_release.py b/tools/verify_release.py index a3fe5f1a..16b87c45 100644 --- a/tools/verify_release.py +++ b/tools/verify_release.py @@ -4,7 +4,8 @@ Two modes: hash Walk a release directory and write a *release hash manifest*: every - .parquet/.json file (relative path, bytes, sha256). Always complete — + .parquet/.json file except *.manifest.json build sidecars (relative + path, bytes, sha256). Always complete — there is deliberately no filter, so a manifest can never look authoritative while omitting files. @@ -28,6 +29,8 @@ that redirects to the origin is not a copy). Bodies are requested unencoded (Accept-Encoding: identity) and any Content-Encoding fails the file. A wrong HEAD size fails fast before the body is fetched. + --dir checks refuse a symlink at the file itself; an ancestor + directory symlink that still resolves inside --dir is accepted. This is the reproducibility programme's step 8 (REPRODUCIBLE_PIPELINE_PLAN_2026-08-25.md): a build's hashes are recorded once, and any copy — R2, a mirror, a Zenodo deposit @@ -51,9 +54,31 @@ CHUNK = 1 << 20 EXTS = (".parquet", ".json") SCHEMA = "release_hashes/1" -# Relative paths a manifest may contain: plain segments, '/' separators, no '..', -# no absolute paths, no backslashes, no URL-significant characters. -PATH_RE = re.compile(r"^(?!/)(?!.*(^|/)\.\.(/|$))[A-Za-z0-9._\-/]+$") +# Relative paths a manifest may contain: '/'-separated segments of +# [A-Za-z0-9._-], no empty/'.'/'..' segments, no leading slash, no trailing +# slash, nothing URL- or shell-significant. +SEG_RE = re.compile(r"[A-Za-z0-9._\-]+") + + +def valid_rel_path(rel): + if not isinstance(rel, str) or not rel or rel.startswith("/") or rel.endswith("/"): + return False + segs = rel.split("/") + return all(SEG_RE.fullmatch(seg) and seg not in (".", "..") for seg in segs) + + +def resolved_inside(path, root): + """True if realpath(path) is root or beneath it.""" + rp = os.path.realpath(path) + return rp == root or rp.startswith(root + os.sep) + + +def parse_base(base): + """Validate --base: http(s), a host, no query/fragment; return it normalised.""" + u = urllib.parse.urlsplit(base) + if u.scheme not in ("http", "https") or not u.netloc or u.query or u.fragment: + raise ValueError(f"--base must be http(s)://host[/path] without query or fragment: {base!r}") + return urllib.parse.urlunsplit((u.scheme, u.netloc, u.path.rstrip("/"), "", "")) # data.isamples.org sits behind Cloudflare, which answers urllib's default # "Python-urllib" agent with 403; identify ourselves instead. UA = "isamples-verify-release/1 (+https://github.com/isamplesorg/isamplesorg.github.io)" @@ -109,10 +134,28 @@ def fetch_hash(url, expected_bytes, timeout): def cmd_hash(args): root = os.path.realpath(args.dir) + if not os.path.isdir(root): + print(f"ERROR: not a directory: {args.dir}", file=sys.stderr) + return 2 + if args.out and resolved_inside(os.path.dirname(os.path.realpath(args.out)) or ".", root): + print("ERROR: --out must not be inside --dir (it would become an unlisted or self-invalidating file)", file=sys.stderr) + return 2 files = {} skipped_links = [] - for dirpath, dirnames, names in os.walk(root, followlinks=False): - dirnames.sort() + + def _walk_error(err): # an unreadable subtree would silently shrink the manifest + raise err + + try: + walk = list(os.walk(root, followlinks=False, onerror=_walk_error)) + except OSError as e: + print(f"ERROR: cannot read the whole release dir: {e}", file=sys.stderr) + return 2 + for dirpath, dirnames, names in walk: + if os.path.islink(dirpath) or any(os.path.islink(os.path.join(dirpath, d)) for d in dirnames): + for d in dirnames: + if os.path.islink(os.path.join(dirpath, d)): + skipped_links.append(os.path.relpath(os.path.join(dirpath, d), root) + "/") for name in sorted(names): if not name.endswith(EXTS) or name.endswith(".manifest.json"): continue @@ -121,7 +164,7 @@ def cmd_hash(args): if os.path.islink(full): skipped_links.append(rel) # a symlink is not release content continue - if not PATH_RE.match(rel): + if not valid_rel_path(rel): print(f"ERROR: path not representable in a manifest: {rel}", file=sys.stderr) return 2 with open(full, "rb") as f: @@ -129,7 +172,7 @@ def cmd_hash(args): files[rel] = {"bytes": n, "sha256": digest} print(f" {digest[:12]} {n:>12,} {rel}") if skipped_links: - print(f"ERROR: {len(skipped_links)} symlinked file(s) in the release dir (not hashed): " + print(f"ERROR: {len(skipped_links)} symlinked entr(y/ies) in the release dir (a manifest describes real files only): " + ", ".join(skipped_links[:5]), file=sys.stderr) return 2 if not files: @@ -155,26 +198,35 @@ def cmd_hash(args): def load_manifest(path): - with open(path) as fh: - doc = json.load(fh) + def _int(v): + return isinstance(v, int) and not isinstance(v, bool) and v >= 0 + + with open(path, "rb") as fh: + raw = fh.read() + doc = json.loads(raw) + if not isinstance(doc, dict): + raise ValueError("manifest is not a JSON object") if doc.get("schema") != SCHEMA: raise ValueError(f"unexpected manifest schema {doc.get('schema')!r} (want {SCHEMA})") + if not isinstance(doc.get("release_id"), str) or not doc["release_id"]: + raise ValueError("missing release_id") files = doc.get("files") if not isinstance(files, dict) or not files: raise ValueError("manifest has no files") - if doc.get("file_count") != len(files): - raise ValueError(f"file_count {doc.get('file_count')} != {len(files)} entries") + if not _int(doc.get("file_count")) or doc["file_count"] != len(files): + raise ValueError(f"file_count {doc.get('file_count')!r} != {len(files)} entries") total = 0 for rel, e in files.items(): - if not PATH_RE.match(rel): + if not valid_rel_path(rel): raise ValueError(f"bad path in manifest: {rel!r}") - if not (isinstance(e.get("bytes"), int) and e["bytes"] >= 0): - raise ValueError(f"bad bytes for {rel}") + if not isinstance(e, dict) or not _int(e.get("bytes")): + raise ValueError(f"bad entry for {rel}") if not (isinstance(e.get("sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", e["sha256"])): raise ValueError(f"bad sha256 for {rel}") total += e["bytes"] - if doc.get("total_bytes") != total: - raise ValueError(f"total_bytes {doc.get('total_bytes')} != sum {total}") + if not _int(doc.get("total_bytes")) or doc["total_bytes"] != total: + raise ValueError(f"total_bytes {doc.get('total_bytes')!r} != sum {total}") + doc["_manifest_sha256"] = hashlib.sha256(raw).hexdigest() return doc @@ -188,9 +240,21 @@ def cmd_check(args): print(f"ERROR: manifest rejected: {e}", file=sys.stderr) return 2 files = doc["files"] - base = args.base.rstrip("/") if args.base else None + try: + base = parse_base(args.base) if args.base else None + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 root = os.path.realpath(args.dir) if args.dir else None - ok, problems, skipped = 0, [], [] + if root and not os.path.isdir(root): + print(f"ERROR: not a directory: {args.dir}", file=sys.stderr) + return 2 + if args.report: + rp = os.path.realpath(args.report) + if (root and resolved_inside(rp, root)) or rp == os.path.realpath(args.manifest): + print("ERROR: --report must not be inside --dir nor alias the manifest", file=sys.stderr) + return 2 + checked, problems, skipped = [], [], [] for rel, exp in files.items(): if (args.only and not fnmatch.fnmatch(rel, args.only)) or (args.skip_prefix and rel.startswith(args.skip_prefix)): skipped.append(rel) @@ -229,9 +293,10 @@ def cmd_check(args): problems.append(("SHA256", rel, f"{digest[:12]} != {exp['sha256'][:12]}")) print(f" SHA256 {rel} {digest[:12]}… != {exp['sha256'][:12]}…") else: - ok += 1 + checked.append(rel) if args.verbose: print(f" ok {rel}") + ok = len(checked) target = base or root if problems or ok == 0: verdict, code = "FAILED", 1 @@ -246,8 +311,10 @@ def cmd_check(args): with open(args.report, "w") as fh: json.dump({"release_id": doc["release_id"], "target": target, "verdict": verdict, "exit_code": code, "checked_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), - "manifest": os.path.abspath(args.manifest), "only": args.only, "skip_prefix": args.skip_prefix, - "ok": ok, "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in problems], + "manifest": os.path.abspath(args.manifest), "manifest_sha256": doc["_manifest_sha256"], + "only": args.only, "skip_prefix": args.skip_prefix, + "ok": ok, "verified_files": checked, + "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in problems], "skipped": skipped}, fh, indent=1) return code From 9ba88b5a3bdacddb6b2f73067aa4ecd2d2401a01 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 16:01:16 -0400 Subject: [PATCH 4/6] verify_release.py: hard-link-safe writers (samefile + atomic temp/replace), commonpath containment, non-empty release id, 'matched when read' contract (Codex round 3) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/verify_release.py | 69 ++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/tools/verify_release.py b/tools/verify_release.py index 16b87c45..1647798c 100644 --- a/tools/verify_release.py +++ b/tools/verify_release.py @@ -5,9 +5,9 @@ hash Walk a release directory and write a *release hash manifest*: every .parquet/.json file except *.manifest.json build sidecars (relative - path, bytes, sha256). Always complete — - there is deliberately no filter, so a manifest can never look - authoritative while omitting files. + path, bytes, sha256). Complete for a quiescent directory — there is + deliberately no filter, so a manifest can never look authoritative + while omitting files. python3 tools/verify_release.py hash --dir ~/Data/iSample/pqg_refining/202609/publish \ --release-id isamples_202609 --out provenance/isamples_202609/release_hashes.json @@ -25,7 +25,11 @@ PARTIAL (3) every *checked* file matches but --only/--skip-prefix left some unchecked; exit 0 only with --allow-partial FAILED (1) a mismatch, a missing file, or an operational error - Files are streamed, never stored. Redirects are refused (a "mirror" + "Matches" means: matched when read. Files are checked one after + another; a target that changes while the run is in progress (or a + directory being written while `hash` walks it) is outside the + contract — verify quiescent, versioned copies. Files are streamed, + never stored. Redirects are refused (a "mirror" that redirects to the origin is not a copy). Bodies are requested unencoded (Accept-Encoding: identity) and any Content-Encoding fails the file. A wrong HEAD size fails fast before the body is fetched. @@ -47,6 +51,7 @@ import os import re import sys +import tempfile import urllib.error import urllib.parse import urllib.request @@ -69,14 +74,37 @@ def valid_rel_path(rel): def resolved_inside(path, root): """True if realpath(path) is root or beneath it.""" - rp = os.path.realpath(path) - return rp == root or rp.startswith(root + os.sep) + try: + return os.path.commonpath([os.path.realpath(path), root]) == root + except ValueError: # different drives (Windows) + return False + + +def same_file(a, b): + """True if two paths name the same inode (catches hard links, not just symlinks).""" + try: + return os.path.samefile(a, b) + except OSError: + return False + + +def write_json_atomic(path, obj): + """Write via a temp file in the destination directory, then os.replace().""" + d = os.path.dirname(os.path.abspath(path)) or "." + fd, tmp = tempfile.mkstemp(prefix=os.path.basename(path) + ".", suffix=".tmp", dir=d) + try: + with os.fdopen(fd, "w") as fh: + json.dump(obj, fh, indent=1) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.remove(tmp) def parse_base(base): """Validate --base: http(s), a host, no query/fragment; return it normalised.""" u = urllib.parse.urlsplit(base) - if u.scheme not in ("http", "https") or not u.netloc or u.query or u.fragment: + if u.scheme not in ("http", "https") or not u.netloc or u.query or u.fragment or "?" in base or "#" in base: raise ValueError(f"--base must be http(s)://host[/path] without query or fragment: {base!r}") return urllib.parse.urlunsplit((u.scheme, u.netloc, u.path.rstrip("/"), "", "")) # data.isamples.org sits behind Cloudflare, which answers urllib's default @@ -137,6 +165,9 @@ def cmd_hash(args): if not os.path.isdir(root): print(f"ERROR: not a directory: {args.dir}", file=sys.stderr) return 2 + if not args.release_id.strip(): + print("ERROR: --release-id must be non-empty", file=sys.stderr) + return 2 if args.out and resolved_inside(os.path.dirname(os.path.realpath(args.out)) or ".", root): print("ERROR: --out must not be inside --dir (it would become an unlisted or self-invalidating file)", file=sys.stderr) return 2 @@ -167,8 +198,15 @@ def _walk_error(err): # an unreadable subtree would silently shrink the if not valid_rel_path(rel): print(f"ERROR: path not representable in a manifest: {rel}", file=sys.stderr) return 2 - with open(full, "rb") as f: - n, digest = hash_and_count(f) + if args.out and same_file(full, args.out): + print(f"ERROR: --out is the same file as release content {rel}", file=sys.stderr) + return 2 + try: + with open(full, "rb") as f: + n, digest = hash_and_count(f) + except OSError as e: + print(f"ERROR: cannot read {rel}: {e}", file=sys.stderr) + return 2 files[rel] = {"bytes": n, "sha256": digest} print(f" {digest[:12]} {n:>12,} {rel}") if skipped_links: @@ -189,8 +227,7 @@ def _walk_error(err): # an unreadable subtree would silently shrink the } if args.out: os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True) - with open(args.out, "w") as fh: - json.dump(doc, fh, indent=1) + write_json_atomic(args.out, doc) print(f"wrote {args.out}: {len(files)} files, {doc['total_bytes']/1e6:.1f} MB") else: json.dump(doc, sys.stdout, indent=1) @@ -251,9 +288,12 @@ def cmd_check(args): return 2 if args.report: rp = os.path.realpath(args.report) - if (root and resolved_inside(rp, root)) or rp == os.path.realpath(args.manifest): + if (root and resolved_inside(rp, root)) or same_file(args.report, args.manifest) or rp == os.path.realpath(args.manifest): print("ERROR: --report must not be inside --dir nor alias the manifest", file=sys.stderr) return 2 + if root and any(same_file(args.report, os.path.join(root, rel)) for rel in files): + print("ERROR: --report is the same file (hard link) as a listed release file", file=sys.stderr) + return 2 checked, problems, skipped = [], [], [] for rel, exp in files.items(): if (args.only and not fnmatch.fnmatch(rel, args.only)) or (args.skip_prefix and rel.startswith(args.skip_prefix)): @@ -308,14 +348,13 @@ def cmd_check(args): f"{len(skipped)} skipped (of {len(files)} listed)" + ("" if not skipped else " [skipped files were NOT checked]")) if args.report: - with open(args.report, "w") as fh: - json.dump({"release_id": doc["release_id"], "target": target, "verdict": verdict, "exit_code": code, + write_json_atomic(args.report, {"release_id": doc["release_id"], "target": target, "verdict": verdict, "exit_code": code, "checked_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), "manifest": os.path.abspath(args.manifest), "manifest_sha256": doc["_manifest_sha256"], "only": args.only, "skip_prefix": args.skip_prefix, "ok": ok, "verified_files": checked, "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in problems], - "skipped": skipped}, fh, indent=1) + "skipped": skipped}) return code From 1ebf74bbf52bdfe69721bed7032a3b33b6d3ed3e Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 16:06:01 -0400 Subject: [PATCH 5/6] verify_release.py: destination checks ignore the final component (symlinks), report written before the verdict and its failure is FAILED (Codex round 4) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/verify_release.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tools/verify_release.py b/tools/verify_release.py index 1647798c..27043e18 100644 --- a/tools/verify_release.py +++ b/tools/verify_release.py @@ -88,6 +88,14 @@ def same_file(a, b): return False +def dest_inside(path, root): + """True if the destination's PARENT resolves inside root, or the destination + itself is a symlink (os.replace would swap the link, creating the file wherever + the parent really is) — evaluated without following the final component.""" + parent = os.path.realpath(os.path.dirname(os.path.abspath(path)) or ".") + return resolved_inside(parent, root) or os.path.islink(path) + + def write_json_atomic(path, obj): """Write via a temp file in the destination directory, then os.replace().""" d = os.path.dirname(os.path.abspath(path)) or "." @@ -168,8 +176,8 @@ def cmd_hash(args): if not args.release_id.strip(): print("ERROR: --release-id must be non-empty", file=sys.stderr) return 2 - if args.out and resolved_inside(os.path.dirname(os.path.realpath(args.out)) or ".", root): - print("ERROR: --out must not be inside --dir (it would become an unlisted or self-invalidating file)", file=sys.stderr) + if args.out and (dest_inside(args.out, root) or os.path.islink(args.out)): + print("ERROR: --out must not be inside --dir nor be a symlink (it would become an unlisted or self-invalidating file)", file=sys.stderr) return 2 files = {} skipped_links = [] @@ -245,7 +253,7 @@ def _int(v): raise ValueError("manifest is not a JSON object") if doc.get("schema") != SCHEMA: raise ValueError(f"unexpected manifest schema {doc.get('schema')!r} (want {SCHEMA})") - if not isinstance(doc.get("release_id"), str) or not doc["release_id"]: + if not isinstance(doc.get("release_id"), str) or not doc["release_id"].strip(): raise ValueError("missing release_id") files = doc.get("files") if not isinstance(files, dict) or not files: @@ -288,8 +296,9 @@ def cmd_check(args): return 2 if args.report: rp = os.path.realpath(args.report) - if (root and resolved_inside(rp, root)) or same_file(args.report, args.manifest) or rp == os.path.realpath(args.manifest): - print("ERROR: --report must not be inside --dir nor alias the manifest", file=sys.stderr) + if (root and dest_inside(args.report, root)) or os.path.islink(args.report) \ + or same_file(args.report, args.manifest) or rp == os.path.realpath(args.manifest): + print("ERROR: --report must not be inside --dir, be a symlink, nor alias the manifest", file=sys.stderr) return 2 if root and any(same_file(args.report, os.path.join(root, rel)) for rel in files): print("ERROR: --report is the same file (hard link) as a listed release file", file=sys.stderr) @@ -344,17 +353,21 @@ def cmd_check(args): verdict, code = "PARTIAL", (0 if args.allow_partial else 3) else: verdict, code = "VERIFIED", 0 - print(f"\n{verdict}: {doc['release_id']} on {target} — {ok} ok, {len(problems)} problem(s), " - f"{len(skipped)} skipped (of {len(files)} listed)" - + ("" if not skipped else " [skipped files were NOT checked]")) if args.report: - write_json_atomic(args.report, {"release_id": doc["release_id"], "target": target, "verdict": verdict, "exit_code": code, + try: + write_json_atomic(args.report, {"release_id": doc["release_id"], "target": target, "verdict": verdict, "exit_code": code, "checked_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), "manifest": os.path.abspath(args.manifest), "manifest_sha256": doc["_manifest_sha256"], "only": args.only, "skip_prefix": args.skip_prefix, "ok": ok, "verified_files": checked, "problems": [{"kind": k, "file": f, "detail": d} for k, f, d in problems], "skipped": skipped}) + except OSError as e: + print(f" ERROR --report {args.report}: {e}") + verdict, code = "FAILED", 1 # an operational error is a failed run, whatever the files said + print(f"\n{verdict}: {doc['release_id']} on {target} — {ok} ok, {len(problems)} problem(s), " + f"{len(skipped)} skipped (of {len(files)} listed)" + + ("" if not skipped else " [skipped files were NOT checked]")) return code From 4ffe2c6f1c1b294dadef42ecf51b5c20b27ad700 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 16:09:37 -0400 Subject: [PATCH 6/6] verify_release.py: hash requires --out (Codex round 5) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/verify_release.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tools/verify_release.py b/tools/verify_release.py index 27043e18..785ea9d1 100644 --- a/tools/verify_release.py +++ b/tools/verify_release.py @@ -233,12 +233,9 @@ def _walk_error(err): # an unreadable subtree would silently shrink the "total_bytes": sum(f["bytes"] for f in files.values()), "files": dict(sorted(files.items())), } - if args.out: - os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True) - write_json_atomic(args.out, doc) - print(f"wrote {args.out}: {len(files)} files, {doc['total_bytes']/1e6:.1f} MB") - else: - json.dump(doc, sys.stdout, indent=1) + os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True) + write_json_atomic(args.out, doc) + print(f"wrote {args.out}: {len(files)} files, {doc['total_bytes']/1e6:.1f} MB") return 0 @@ -377,7 +374,7 @@ def main(): h = sub.add_parser("hash", help="write a complete release hash manifest from a directory") h.add_argument("--dir", required=True) h.add_argument("--release-id", required=True) - h.add_argument("--out") + h.add_argument("--out", required=True, help="manifest path (progress goes to stdout, so the manifest is always a file)") h.set_defaults(fn=cmd_hash) c = sub.add_parser("check", help="verify a host or directory against a release hash manifest") c.add_argument("--manifest", required=True)