correct-reports: rewrite the fp32 reports already on the Hub

pipeline/__main__.py

@@ -255,6 +255,17 @@ def _publish_hf(args) -> int:
     return 0
 
 
+def _correct_reports(args) -> int:
+    import subprocess
+
+    from pipeline import correct
+
+    commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
+    for repo in args.repos:
+        print(json.dumps(correct.correct_repo(repo, commit or "unknown", dry_run=not args.apply), indent=2))
+    return 0
+
+
 def _publish_release(args) -> int:
     from pipeline import publish
 
@@ -432,6 +443,15 @@ def main(argv: list[str] | None = None) -> int:
     )
     watch.set_defaults(func=_watch)
 
+    fix = commands.add_parser(
+        "correct-reports", help="rewrite fp32 reports' recipe and sizing fields on the Hub; never touches a .pte"
+    )
+    fix.add_argument(
+        "repos", nargs="+", help="output repos, e.g. experimentalmachines/SmolLM2-360M-Instruct-ExecuTorch"
+    )
+    fix.add_argument("--apply", action="store_true", help="commit the corrections (default: report what would change)")
+    fix.set_defaults(func=_correct_reports)
+
     for name, func, text in (
         ("publish-hf", _publish_hf, "commit one backend folder to the output HF repo"),
         ("publish-release", _publish_release, "attach one backend's files to a GitHub release"),

pipeline/correct.py

@@ -0,0 +1,138 @@
+"""Correct what published export reports say about an fp32 export, without touching any .pte.
+
+Exports made with the fp32 recipe before it was described correctly (fixed in the commit that
+added ``export_xnnpack.recipe_fields`` and the per-recipe sizing) carry the 8da4w wording and
+sizing: int4 codes, a group size, "4-bit weights", and a phone estimate for 4-bit linears. The
+files themselves are right. This rewrites only those report fields, regenerates the folder's
+config.json (the app reads its ``quantization`` label and ``fits_phone_budget``) and the README
+from the corrected reports, and commits them pinned to the revision it read, so the .pte files,
+their hashes and every pin against them stay as they are.
+
+A report is corrected only when the old figures can be reproduced exactly from the 8da4w
+sizing with the configured runtime overhead: that proves which inputs the export used, so
+the fp32 figures replace them like for like. Anything else is refused, not guessed.
+"""
+
+from __future__ import annotations
+
+import copy
+import dataclasses
+import datetime
+import json
+from pathlib import Path
+
+from pipeline import families, hub, manifest, settings, sizing
+from pipeline.export_xnnpack import recipe_fields
+
+FIELDS = (
+    "recipe.group_size",
+    "recipe.int4_codes",
+    "recipe.label",
+    "recipe.description",
+    "window.device_resident_bytes",
+    "window.fits_phone_budget",
+    "window.table[].device_resident_bytes",
+    "window.table[].fits_device",
+    "estimates.pte_bytes_estimate",
+)
+
+
+class CannotCorrect(ValueError):
+    pass
+
+
+def corrected(report: dict, arch: sizing.Architecture, cfg, commit: str, today: str) -> dict | None:
+    """The report with its fp32 description and sizing fixed, or None when it needs nothing."""
+    recipe = report.get("recipe") or {}
+    if report.get("backend") != "xnnpack" or recipe.get("qmode") != "fp32":
+        return None
+    window = report["window"]
+    overhead = cfg.runtime_overhead_bytes
+    # The old figures must be exactly the 8da4w sizing of this architecture and overhead.
+    rows = [window, *window["table"]]
+    for row in rows:
+        expected = sizing.device_resident_bytes(arch, row["context"], overhead, "8da4w")
+        fp32 = sizing.device_resident_bytes(arch, row["context"], overhead, "fp32")
+        if row["device_resident_bytes"] not in (expected, fp32):
+            raise CannotCorrect(
+                f"{report['files'][0]['path']}: resident bytes at {row['context']} are "
+                f"{row['device_resident_bytes']:,}, neither the 8da4w figure {expected:,} nor the fp32 one {fp32:,}"
+            )
+    new = copy.deepcopy(report)
+    base = dataclasses.replace(
+        cfg.xnnpack, qmode="fp32", embedding_quantize=recipe["embedding_quantize"], group_size=recipe["group_size"]
+    )
+    new["recipe"].update(
+        recipe_fields(
+            base, "fp32", False, "xnnpack", recipe["prefill_chunk"], executorch=report["toolchain"]["executorch"]
+        )
+    )
+    budget = window["phone_budget_bytes"]
+    for row in (new["window"], *new["window"]["table"]):
+        row["device_resident_bytes"] = sizing.device_resident_bytes(arch, row["context"], overhead, "fp32")
+    new["window"]["fits_phone_budget"] = new["window"]["device_resident_bytes"] <= budget
+    for row in new["window"]["table"]:
+        row["fits_device"] = row["device_resident_bytes"] <= budget
+    if "estimates" in new:
+        new["estimates"]["pte_bytes_estimate"] = sizing.pte_bytes_estimate(arch, window["context"], "fp32")
+    if new == report:
+        return None
+    new.setdefault("corrections", []).append(
+        {
+            "date": today,
+            "by": f"execupack {commit}",
+            "fields": list(FIELDS),
+            "reason": "fp32 export described and sized as 8da4w; the .pte is unchanged",
+        }
+    )
+    return new
+
+
+def correct_repo(repo_id: str, commit: str, dry_run: bool = True) -> dict:
+    """Correct every fp32 report in one Hub repo; return what changed (and the commit URL)."""
+    from huggingface_hub import CommitOperationAdd, hf_hub_download
+
+    cfg = settings.load()
+    hf = hub.api()
+    info = hf.model_info(repo_id, expand=["sha", "siblings"])
+    from pipeline.publish import is_report
+
+    paths = sorted(s.rfilename for s in info.siblings or [] if is_report(s.rfilename))
+    reports = {
+        p: json.loads(Path(hf_hub_download(repo_id, p, revision=info.sha, token=hf.token)).read_text()) for p in paths
+    }
+    today = datetime.date.today().isoformat()
+    changed: dict[str, dict] = {}
+    archs: dict[tuple, sizing.Architecture] = {}
+    for path, report in reports.items():
+        key = (report["source"]["id"], report["source"]["sha"])
+        if report.get("recipe", {}).get("qmode") == "fp32" and key not in archs:
+            source = hub.fetch(*key)
+            archs[key] = families.architecture(source.config, source.total_params)
+        fixed = corrected(report, archs[key], cfg, commit, today) if key in archs else None
+        if fixed is not None:
+            changed[path] = fixed
+    result = {"repo": repo_id, "revision": info.sha, "corrected": sorted(changed), "commit_url": None}
+    if not changed or dry_run:
+        return result
+    reports.update(changed)
+    all_reports = list(reports.values())
+    operations = [
+        CommitOperationAdd(path_in_repo=p, path_or_fileobj=(json.dumps(r, indent=2) + "\n").encode())
+        for p, r in sorted(changed.items())
+    ]
+    for folder in sorted({p.rsplit("/", 1)[0] for p in changed}):
+        folder_reports = [r for p, r in reports.items() if p.rsplit("/", 1)[0] == folder]
+        config = json.dumps(manifest.backend_config(folder_reports), indent=2) + "\n"
+        operations.append(CommitOperationAdd(path_in_repo=f"{folder}/config.json", path_or_fileobj=config.encode()))
+    license_files = sorted({f for r in all_reports for f in r["source"].get("license_files", [])})
+    readme = manifest.readme(repo_id, all_reports, list(cfg.hub_tags), license_files)
+    operations.append(CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme.encode("utf-8")))
+    commit_info = hf.create_commit(
+        repo_id,
+        operations,
+        commit_message=f"correct fp32 report fields ({len(changed)} reports); no .pte changed",
+        parent_commit=info.sha,
+    )
+    result["commit_url"] = commit_info.commit_url
+    return result

pipeline/export_xnnpack.py

@@ -138,7 +138,9 @@ def pick_runner(
     return None
 
 
-def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int) -> dict:
+def recipe_fields(
+    recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int, executorch: str | None = None
+) -> dict:
     """The report's description of how the weights are stored, true to the recipe built.
 
     8da4w: dynamic int8 activations and int4 weights in groups, the codes rounded to nearest or
@@ -147,6 +149,8 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_
     every reader of the report, and of the README built from it, that its weights were 4-bit.
     """
     delegate = "XNNPACK with extended ops" if backend == "xnnpack" else "the Vulkan delegate"
+    # A correction describes the toolchain the file was built with, not the one running now.
+    version = executorch or toolchain()["executorch"]
     common = {
         "qmode": qmode_label,
         "embedding_quantize": recipe.embedding_quantize,
@@ -160,7 +164,7 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_
             "int4_codes": None,
             "label": "fp32 linears, int8 embeddings",
             "description": (
-                f"ExecuTorch {toolchain()['executorch']} `export_llm`: fp32 weights and activations in every "
+                f"ExecuTorch {version} `export_llm`: fp32 weights and activations in every "
                 f"linear (not quantized), int8 per-channel embeddings, {delegate}, prefill chunk "
                 f"{prefill_chunk}, fp32 KV cache."
             ),
@@ -174,7 +178,7 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_
             + (", int4 codes solved with GPTQ" if solved else ", int4 codes rounded to nearest")
         ),
         "description": (
-            f"ExecuTorch {toolchain()['executorch']} `export_llm`: 8-bit dynamic activations "
+            f"ExecuTorch {version} `export_llm`: 8-bit dynamic activations "
             f"and 4-bit weights in groups of {recipe.group_size}, int8 per-channel embeddings, "
             f"{delegate}, prefill chunk {prefill_chunk}, fp32 KV cache."
         ),

tests/test_correct.py

@@ -0,0 +1,86 @@
+"""pipeline/correct.py: an fp32 report's description and sizing corrected, nothing else, never guessed."""
+
+import copy
+import dataclasses
+
+import pytest
+from conftest import TOTAL_PARAMS, hf_config
+
+from pipeline import correct, families, settings, sizing
+from pipeline.export_xnnpack import recipe_fields
+
+MODEL = "HuggingFaceTB/SmolLM2-360M-Instruct"
+
+
+def _published(context=8192):
+    """A report as the fp32 recipe wrote it before the fix: 8da4w wording and sizing."""
+    cfg = settings.load()
+    arch = families.architecture(hf_config(MODEL), TOTAL_PARAMS[MODEL])
+    old = recipe_fields(cfg.xnnpack, "8da4w", False, "xnnpack", 2048, executorch="1.4.0")
+    old.update({"qmode": "fp32", "label": "fp32-g32, int8 embeddings, int4 codes rounded to nearest"})
+    table = [
+        {
+            "context": c,
+            "device_resident_bytes": sizing.device_resident_bytes(arch, c, cfg.runtime_overhead_bytes),
+            "fits_device": True,
+            "export_peak_bytes": 1,
+            "fits_host": True,
+        }
+        for c in (32768, 16384, 8192, 4096, 2048)
+    ]
+    report = {
+        "backend": "xnnpack",
+        "toolchain": {"executorch": "1.4.0"},
+        "source": {"id": MODEL, "sha": "a" * 40},
+        "recipe": {"export_llm_qmode": None, "embedding_hqq": False, **old},
+        "window": {
+            "context": context,
+            "device_resident_bytes": sizing.device_resident_bytes(arch, context, cfg.runtime_overhead_bytes),
+            "phone_budget_bytes": cfg.device_budget_bytes,
+            "fits_phone_budget": True,
+            "table": table,
+        },
+        "estimates": {"pte_bytes_estimate": sizing.pte_bytes_estimate(arch, context)},
+        "files": [{"path": "xnnpack/SmolLM2-360M-Instruct-fp32-8k.pte", "bytes": 1499480448, "sha256": "f" * 64}],
+    }
+    return report, arch, cfg
+
+
+def test_an_fp32_report_is_corrected_and_only_its_described_fields_change():
+    report, arch, cfg = _published()
+    fixed = correct.corrected(report, arch, cfg, "abc1234", "2026-10-04")
+    assert fixed["recipe"]["int4_codes"] is None and fixed["recipe"]["group_size"] is None
+    assert "4-bit" not in fixed["recipe"]["description"] and "ExecuTorch 1.4.0" in fixed["recipe"]["description"]
+    assert fixed["window"]["device_resident_bytes"] == sizing.device_resident_bytes(
+        arch, 8192, cfg.runtime_overhead_bytes, "fp32"
+    )
+    assert fixed["estimates"]["pte_bytes_estimate"] == sizing.pte_bytes_estimate(arch, 8192, "fp32")
+    assert fixed["files"] == report["files"]  # the .pte is never touched
+    assert fixed["corrections"][-1]["by"] == "execupack abc1234"
+    # Everything outside the listed fields is as it was.
+    trimmed = copy.deepcopy(fixed)
+    for key in ("group_size", "int4_codes", "label", "description"):
+        trimmed["recipe"][key] = report["recipe"][key]
+    trimmed["window"] = report["window"]
+    trimmed["estimates"] = report["estimates"]
+    del trimmed["corrections"]
+    assert trimmed == report
+    # Correcting twice changes nothing more.
+    assert correct.corrected(fixed, arch, cfg, "abc1234", "2026-10-04") is None
+
+
+def test_8da4w_reports_are_left_alone():
+    report, arch, cfg = _published()
+    report["recipe"]["qmode"] = "8da4w-gptq"
+    assert correct.corrected(report, arch, cfg, "x", "d") is None
+
+
+def test_figures_that_cannot_be_reproduced_are_refused():
+    report, arch, cfg = _published()
+    report["window"]["device_resident_bytes"] += 1
+    with pytest.raises(correct.CannotCorrect, match="neither"):
+        correct.corrected(report, arch, cfg, "x", "d")
+    report, arch, cfg = _published()
+    other = dataclasses.replace(cfg, runtime_overhead_bytes=cfg.runtime_overhead_bytes + 1)
+    with pytest.raises(correct.CannotCorrect):
+        correct.corrected(report, arch, other, "x", "d")