From 6d41fd246a9b529505f00f114b35df9522fba6b6 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Sun, 4 Oct 2026 20:23:38 +0800 Subject: [PATCH] correct-reports: rewrite the fp32 reports already on the Hub, because they still carry the 8da4w recipe and sizing `python -m pipeline correct-reports [--apply]` rewrites, in fp32 xnnpack reports only, the recipe wording (group size and int4 codes null, fp32 label and description, named with the toolchain the file was built with) and the sizing (window, table rows, pte estimate), records a `corrections` entry, regenerates the folder's config.json (the app reads its quantization label and fits_phone_budget) and the README, and commits pinned to the revision it read. No .pte is touched. A report is corrected only if its old figures reproduce exactly from the 8da4w sizing with the configured overhead; otherwise it is refused. Dry run on experimentalmachines/SmolLM2-360M-Instruct-ExecuTorch: the 5 xnnpack reports, all reproduced. The corrected .pte estimates are within 1 to 2 percent of the files (1,511,281,131 B estimated against 1,496,334,720 B at 2k); the old ones were 255,300,075 B. Co-Authored-By: Claude Opus 5.5 --- pipeline/__main__.py | 20 ++++++ pipeline/correct.py | 138 +++++++++++++++++++++++++++++++++++++ pipeline/export_xnnpack.py | 10 ++- tests/test_correct.py | 86 +++++++++++++++++++++++ 4 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 pipeline/correct.py create mode 100644 tests/test_correct.py diff --git a/pipeline/__main__.py b/pipeline/__main__.py index e62f10d..a792ccc 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -255,6 +255,17 @@ def _publish_hf(args) -> int: return 0 +def _correct_reports(args) -> int: + import subprocess + + from pipeline import correct + + commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip() + for repo in args.repos: + print(json.dumps(correct.correct_repo(repo, commit or "unknown", dry_run=not args.apply), indent=2)) + return 0 + + def _publish_release(args) -> int: from pipeline import publish @@ -432,6 +443,15 @@ def main(argv: list[str] | None = None) -> int: ) watch.set_defaults(func=_watch) + fix = commands.add_parser( + "correct-reports", help="rewrite fp32 reports' recipe and sizing fields on the Hub; never touches a .pte" + ) + fix.add_argument( + "repos", nargs="+", help="output repos, e.g. experimentalmachines/SmolLM2-360M-Instruct-ExecuTorch" + ) + fix.add_argument("--apply", action="store_true", help="commit the corrections (default: report what would change)") + fix.set_defaults(func=_correct_reports) + for name, func, text in ( ("publish-hf", _publish_hf, "commit one backend folder to the output HF repo"), ("publish-release", _publish_release, "attach one backend's files to a GitHub release"), diff --git a/pipeline/correct.py b/pipeline/correct.py new file mode 100644 index 0000000..cc8e652 --- /dev/null +++ b/pipeline/correct.py @@ -0,0 +1,138 @@ +"""Correct what published export reports say about an fp32 export, without touching any .pte. + +Exports made with the fp32 recipe before it was described correctly (fixed in the commit that +added ``export_xnnpack.recipe_fields`` and the per-recipe sizing) carry the 8da4w wording and +sizing: int4 codes, a group size, "4-bit weights", and a phone estimate for 4-bit linears. The +files themselves are right. This rewrites only those report fields, regenerates the folder's +config.json (the app reads its ``quantization`` label and ``fits_phone_budget``) and the README +from the corrected reports, and commits them pinned to the revision it read, so the .pte files, +their hashes and every pin against them stay as they are. + +A report is corrected only when the old figures can be reproduced exactly from the 8da4w +sizing with the configured runtime overhead: that proves which inputs the export used, so +the fp32 figures replace them like for like. Anything else is refused, not guessed. +""" + +from __future__ import annotations + +import copy +import dataclasses +import datetime +import json +from pathlib import Path + +from pipeline import families, hub, manifest, settings, sizing +from pipeline.export_xnnpack import recipe_fields + +FIELDS = ( + "recipe.group_size", + "recipe.int4_codes", + "recipe.label", + "recipe.description", + "window.device_resident_bytes", + "window.fits_phone_budget", + "window.table[].device_resident_bytes", + "window.table[].fits_device", + "estimates.pte_bytes_estimate", +) + + +class CannotCorrect(ValueError): + pass + + +def corrected(report: dict, arch: sizing.Architecture, cfg, commit: str, today: str) -> dict | None: + """The report with its fp32 description and sizing fixed, or None when it needs nothing.""" + recipe = report.get("recipe") or {} + if report.get("backend") != "xnnpack" or recipe.get("qmode") != "fp32": + return None + window = report["window"] + overhead = cfg.runtime_overhead_bytes + # The old figures must be exactly the 8da4w sizing of this architecture and overhead. + rows = [window, *window["table"]] + for row in rows: + expected = sizing.device_resident_bytes(arch, row["context"], overhead, "8da4w") + fp32 = sizing.device_resident_bytes(arch, row["context"], overhead, "fp32") + if row["device_resident_bytes"] not in (expected, fp32): + raise CannotCorrect( + f"{report['files'][0]['path']}: resident bytes at {row['context']} are " + f"{row['device_resident_bytes']:,}, neither the 8da4w figure {expected:,} nor the fp32 one {fp32:,}" + ) + new = copy.deepcopy(report) + base = dataclasses.replace( + cfg.xnnpack, qmode="fp32", embedding_quantize=recipe["embedding_quantize"], group_size=recipe["group_size"] + ) + new["recipe"].update( + recipe_fields( + base, "fp32", False, "xnnpack", recipe["prefill_chunk"], executorch=report["toolchain"]["executorch"] + ) + ) + budget = window["phone_budget_bytes"] + for row in (new["window"], *new["window"]["table"]): + row["device_resident_bytes"] = sizing.device_resident_bytes(arch, row["context"], overhead, "fp32") + new["window"]["fits_phone_budget"] = new["window"]["device_resident_bytes"] <= budget + for row in new["window"]["table"]: + row["fits_device"] = row["device_resident_bytes"] <= budget + if "estimates" in new: + new["estimates"]["pte_bytes_estimate"] = sizing.pte_bytes_estimate(arch, window["context"], "fp32") + if new == report: + return None + new.setdefault("corrections", []).append( + { + "date": today, + "by": f"execupack {commit}", + "fields": list(FIELDS), + "reason": "fp32 export described and sized as 8da4w; the .pte is unchanged", + } + ) + return new + + +def correct_repo(repo_id: str, commit: str, dry_run: bool = True) -> dict: + """Correct every fp32 report in one Hub repo; return what changed (and the commit URL).""" + from huggingface_hub import CommitOperationAdd, hf_hub_download + + cfg = settings.load() + hf = hub.api() + info = hf.model_info(repo_id, expand=["sha", "siblings"]) + from pipeline.publish import is_report + + paths = sorted(s.rfilename for s in info.siblings or [] if is_report(s.rfilename)) + reports = { + p: json.loads(Path(hf_hub_download(repo_id, p, revision=info.sha, token=hf.token)).read_text()) for p in paths + } + today = datetime.date.today().isoformat() + changed: dict[str, dict] = {} + archs: dict[tuple, sizing.Architecture] = {} + for path, report in reports.items(): + key = (report["source"]["id"], report["source"]["sha"]) + if report.get("recipe", {}).get("qmode") == "fp32" and key not in archs: + source = hub.fetch(*key) + archs[key] = families.architecture(source.config, source.total_params) + fixed = corrected(report, archs[key], cfg, commit, today) if key in archs else None + if fixed is not None: + changed[path] = fixed + result = {"repo": repo_id, "revision": info.sha, "corrected": sorted(changed), "commit_url": None} + if not changed or dry_run: + return result + reports.update(changed) + all_reports = list(reports.values()) + operations = [ + CommitOperationAdd(path_in_repo=p, path_or_fileobj=(json.dumps(r, indent=2) + "\n").encode()) + for p, r in sorted(changed.items()) + ] + for folder in sorted({p.rsplit("/", 1)[0] for p in changed}): + folder_reports = [r for p, r in reports.items() if p.rsplit("/", 1)[0] == folder] + config = json.dumps(manifest.backend_config(folder_reports), indent=2) + "\n" + operations.append(CommitOperationAdd(path_in_repo=f"{folder}/config.json", path_or_fileobj=config.encode())) + license_files = sorted({f for r in all_reports for f in r["source"].get("license_files", [])}) + readme = manifest.readme(repo_id, all_reports, list(cfg.hub_tags), license_files) + operations.append(CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme.encode("utf-8"))) + commit_info = hf.create_commit( + repo_id, + operations, + commit_message=f"correct fp32 report fields ({len(changed)} reports); no .pte changed", + parent_commit=info.sha, + ) + result["commit_url"] = commit_info.commit_url + return result diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 20c630a..2212784 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -138,7 +138,9 @@ def pick_runner( return None -def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int) -> dict: +def recipe_fields( + recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int, executorch: str | None = None +) -> dict: """The report's description of how the weights are stored, true to the recipe built. 8da4w: dynamic int8 activations and int4 weights in groups, the codes rounded to nearest or @@ -147,6 +149,8 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_ every reader of the report, and of the README built from it, that its weights were 4-bit. """ delegate = "XNNPACK with extended ops" if backend == "xnnpack" else "the Vulkan delegate" + # A correction describes the toolchain the file was built with, not the one running now. + version = executorch or toolchain()["executorch"] common = { "qmode": qmode_label, "embedding_quantize": recipe.embedding_quantize, @@ -160,7 +164,7 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_ "int4_codes": None, "label": "fp32 linears, int8 embeddings", "description": ( - f"ExecuTorch {toolchain()['executorch']} `export_llm`: fp32 weights and activations in every " + f"ExecuTorch {version} `export_llm`: fp32 weights and activations in every " f"linear (not quantized), int8 per-channel embeddings, {delegate}, prefill chunk " f"{prefill_chunk}, fp32 KV cache." ), @@ -174,7 +178,7 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_ + (", int4 codes solved with GPTQ" if solved else ", int4 codes rounded to nearest") ), "description": ( - f"ExecuTorch {toolchain()['executorch']} `export_llm`: 8-bit dynamic activations " + f"ExecuTorch {version} `export_llm`: 8-bit dynamic activations " f"and 4-bit weights in groups of {recipe.group_size}, int8 per-channel embeddings, " f"{delegate}, prefill chunk {prefill_chunk}, fp32 KV cache." ), diff --git a/tests/test_correct.py b/tests/test_correct.py new file mode 100644 index 0000000..3edaf2f --- /dev/null +++ b/tests/test_correct.py @@ -0,0 +1,86 @@ +"""pipeline/correct.py: an fp32 report's description and sizing corrected, nothing else, never guessed.""" + +import copy +import dataclasses + +import pytest +from conftest import TOTAL_PARAMS, hf_config + +from pipeline import correct, families, settings, sizing +from pipeline.export_xnnpack import recipe_fields + +MODEL = "HuggingFaceTB/SmolLM2-360M-Instruct" + + +def _published(context=8192): + """A report as the fp32 recipe wrote it before the fix: 8da4w wording and sizing.""" + cfg = settings.load() + arch = families.architecture(hf_config(MODEL), TOTAL_PARAMS[MODEL]) + old = recipe_fields(cfg.xnnpack, "8da4w", False, "xnnpack", 2048, executorch="1.4.0") + old.update({"qmode": "fp32", "label": "fp32-g32, int8 embeddings, int4 codes rounded to nearest"}) + table = [ + { + "context": c, + "device_resident_bytes": sizing.device_resident_bytes(arch, c, cfg.runtime_overhead_bytes), + "fits_device": True, + "export_peak_bytes": 1, + "fits_host": True, + } + for c in (32768, 16384, 8192, 4096, 2048) + ] + report = { + "backend": "xnnpack", + "toolchain": {"executorch": "1.4.0"}, + "source": {"id": MODEL, "sha": "a" * 40}, + "recipe": {"export_llm_qmode": None, "embedding_hqq": False, **old}, + "window": { + "context": context, + "device_resident_bytes": sizing.device_resident_bytes(arch, context, cfg.runtime_overhead_bytes), + "phone_budget_bytes": cfg.device_budget_bytes, + "fits_phone_budget": True, + "table": table, + }, + "estimates": {"pte_bytes_estimate": sizing.pte_bytes_estimate(arch, context)}, + "files": [{"path": "xnnpack/SmolLM2-360M-Instruct-fp32-8k.pte", "bytes": 1499480448, "sha256": "f" * 64}], + } + return report, arch, cfg + + +def test_an_fp32_report_is_corrected_and_only_its_described_fields_change(): + report, arch, cfg = _published() + fixed = correct.corrected(report, arch, cfg, "abc1234", "2026-10-04") + assert fixed["recipe"]["int4_codes"] is None and fixed["recipe"]["group_size"] is None + assert "4-bit" not in fixed["recipe"]["description"] and "ExecuTorch 1.4.0" in fixed["recipe"]["description"] + assert fixed["window"]["device_resident_bytes"] == sizing.device_resident_bytes( + arch, 8192, cfg.runtime_overhead_bytes, "fp32" + ) + assert fixed["estimates"]["pte_bytes_estimate"] == sizing.pte_bytes_estimate(arch, 8192, "fp32") + assert fixed["files"] == report["files"] # the .pte is never touched + assert fixed["corrections"][-1]["by"] == "execupack abc1234" + # Everything outside the listed fields is as it was. + trimmed = copy.deepcopy(fixed) + for key in ("group_size", "int4_codes", "label", "description"): + trimmed["recipe"][key] = report["recipe"][key] + trimmed["window"] = report["window"] + trimmed["estimates"] = report["estimates"] + del trimmed["corrections"] + assert trimmed == report + # Correcting twice changes nothing more. + assert correct.corrected(fixed, arch, cfg, "abc1234", "2026-10-04") is None + + +def test_8da4w_reports_are_left_alone(): + report, arch, cfg = _published() + report["recipe"]["qmode"] = "8da4w-gptq" + assert correct.corrected(report, arch, cfg, "x", "d") is None + + +def test_figures_that_cannot_be_reproduced_are_refused(): + report, arch, cfg = _published() + report["window"]["device_resident_bytes"] += 1 + with pytest.raises(correct.CannotCorrect, match="neither"): + correct.corrected(report, arch, cfg, "x", "d") + report, arch, cfg = _published() + other = dataclasses.replace(cfg, runtime_overhead_bytes=cfg.runtime_overhead_bytes + 1) + with pytest.raises(correct.CannotCorrect): + correct.corrected(report, arch, other, "x", "d")