diff --git a/pipeline/__main__.py b/pipeline/__main__.py index e62f10d..a792ccc 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -255,6 +255,17 @@ def _publish_hf(args) -> int: return 0 +def _correct_reports(args) -> int: + import subprocess + + from pipeline import correct + + commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip() + for repo in args.repos: + print(json.dumps(correct.correct_repo(repo, commit or "unknown", dry_run=not args.apply), indent=2)) + return 0 + + def _publish_release(args) -> int: from pipeline import publish @@ -432,6 +443,15 @@ def main(argv: list[str] | None = None) -> int: ) watch.set_defaults(func=_watch) + fix = commands.add_parser( + "correct-reports", help="rewrite fp32 reports' recipe and sizing fields on the Hub; never touches a .pte" + ) + fix.add_argument( + "repos", nargs="+", help="output repos, e.g. experimentalmachines/SmolLM2-360M-Instruct-ExecuTorch" + ) + fix.add_argument("--apply", action="store_true", help="commit the corrections (default: report what would change)") + fix.set_defaults(func=_correct_reports) + for name, func, text in ( ("publish-hf", _publish_hf, "commit one backend folder to the output HF repo"), ("publish-release", _publish_release, "attach one backend's files to a GitHub release"), diff --git a/pipeline/correct.py b/pipeline/correct.py new file mode 100644 index 0000000..cc8e652 --- /dev/null +++ b/pipeline/correct.py @@ -0,0 +1,138 @@ +"""Correct what published export reports say about an fp32 export, without touching any .pte. + +Exports made with the fp32 recipe before it was described correctly (fixed in the commit that +added ``export_xnnpack.recipe_fields`` and the per-recipe sizing) carry the 8da4w wording and +sizing: int4 codes, a group size, "4-bit weights", and a phone estimate for 4-bit linears. The +files themselves are right. This rewrites only those report fields, regenerates the folder's +config.json (the app reads its ``quantization`` label and ``fits_phone_budget``) and the README +from the corrected reports, and commits them pinned to the revision it read, so the .pte files, +their hashes and every pin against them stay as they are. + +A report is corrected only when the old figures can be reproduced exactly from the 8da4w +sizing with the configured runtime overhead: that proves which inputs the export used, so +the fp32 figures replace them like for like. Anything else is refused, not guessed. +""" + +from __future__ import annotations + +import copy +import dataclasses +import datetime +import json +from pathlib import Path + +from pipeline import families, hub, manifest, settings, sizing +from pipeline.export_xnnpack import recipe_fields + +FIELDS = ( + "recipe.group_size", + "recipe.int4_codes", + "recipe.label", + "recipe.description", + "window.device_resident_bytes", + "window.fits_phone_budget", + "window.table[].device_resident_bytes", + "window.table[].fits_device", + "estimates.pte_bytes_estimate", +) + + +class CannotCorrect(ValueError): + pass + + +def corrected(report: dict, arch: sizing.Architecture, cfg, commit: str, today: str) -> dict | None: + """The report with its fp32 description and sizing fixed, or None when it needs nothing.""" + recipe = report.get("recipe") or {} + if report.get("backend") != "xnnpack" or recipe.get("qmode") != "fp32": + return None + window = report["window"] + overhead = cfg.runtime_overhead_bytes + # The old figures must be exactly the 8da4w sizing of this architecture and overhead. + rows = [window, *window["table"]] + for row in rows: + expected = sizing.device_resident_bytes(arch, row["context"], overhead, "8da4w") + fp32 = sizing.device_resident_bytes(arch, row["context"], overhead, "fp32") + if row["device_resident_bytes"] not in (expected, fp32): + raise CannotCorrect( + f"{report['files'][0]['path']}: resident bytes at {row['context']} are " + f"{row['device_resident_bytes']:,}, neither the 8da4w figure {expected:,} nor the fp32 one {fp32:,}" + ) + new = copy.deepcopy(report) + base = dataclasses.replace( + cfg.xnnpack, qmode="fp32", embedding_quantize=recipe["embedding_quantize"], group_size=recipe["group_size"] + ) + new["recipe"].update( + recipe_fields( + base, "fp32", False, "xnnpack", recipe["prefill_chunk"], executorch=report["toolchain"]["executorch"] + ) + ) + budget = window["phone_budget_bytes"] + for row in (new["window"], *new["window"]["table"]): + row["device_resident_bytes"] = sizing.device_resident_bytes(arch, row["context"], overhead, "fp32") + new["window"]["fits_phone_budget"] = new["window"]["device_resident_bytes"] <= budget + for row in new["window"]["table"]: + row["fits_device"] = row["device_resident_bytes"] <= budget + if "estimates" in new: + new["estimates"]["pte_bytes_estimate"] = sizing.pte_bytes_estimate(arch, window["context"], "fp32") + if new == report: + return None + new.setdefault("corrections", []).append( + { + "date": today, + "by": f"execupack {commit}", + "fields": list(FIELDS), + "reason": "fp32 export described and sized as 8da4w; the .pte is unchanged", + } + ) + return new + + +def correct_repo(repo_id: str, commit: str, dry_run: bool = True) -> dict: + """Correct every fp32 report in one Hub repo; return what changed (and the commit URL).""" + from huggingface_hub import CommitOperationAdd, hf_hub_download + + cfg = settings.load() + hf = hub.api() + info = hf.model_info(repo_id, expand=["sha", "siblings"]) + from pipeline.publish import is_report + + paths = sorted(s.rfilename for s in info.siblings or [] if is_report(s.rfilename)) + reports = { + p: json.loads(Path(hf_hub_download(repo_id, p, revision=info.sha, token=hf.token)).read_text()) for p in paths + } + today = datetime.date.today().isoformat() + changed: dict[str, dict] = {} + archs: dict[tuple, sizing.Architecture] = {} + for path, report in reports.items(): + key = (report["source"]["id"], report["source"]["sha"]) + if report.get("recipe", {}).get("qmode") == "fp32" and key not in archs: + source = hub.fetch(*key) + archs[key] = families.architecture(source.config, source.total_params) + fixed = corrected(report, archs[key], cfg, commit, today) if key in archs else None + if fixed is not None: + changed[path] = fixed + result = {"repo": repo_id, "revision": info.sha, "corrected": sorted(changed), "commit_url": None} + if not changed or dry_run: + return result + reports.update(changed) + all_reports = list(reports.values()) + operations = [ + CommitOperationAdd(path_in_repo=p, path_or_fileobj=(json.dumps(r, indent=2) + "\n").encode()) + for p, r in sorted(changed.items()) + ] + for folder in sorted({p.rsplit("/", 1)[0] for p in changed}): + folder_reports = [r for p, r in reports.items() if p.rsplit("/", 1)[0] == folder] + config = json.dumps(manifest.backend_config(folder_reports), indent=2) + "\n" + operations.append(CommitOperationAdd(path_in_repo=f"{folder}/config.json", path_or_fileobj=config.encode())) + license_files = sorted({f for r in all_reports for f in r["source"].get("license_files", [])}) + readme = manifest.readme(repo_id, all_reports, list(cfg.hub_tags), license_files) + operations.append(CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme.encode("utf-8"))) + commit_info = hf.create_commit( + repo_id, + operations, + commit_message=f"correct fp32 report fields ({len(changed)} reports); no .pte changed", + parent_commit=info.sha, + ) + result["commit_url"] = commit_info.commit_url + return result diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 20c630a..2212784 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -138,7 +138,9 @@ def pick_runner( return None -def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int) -> dict: +def recipe_fields( + recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int, executorch: str | None = None +) -> dict: """The report's description of how the weights are stored, true to the recipe built. 8da4w: dynamic int8 activations and int4 weights in groups, the codes rounded to nearest or @@ -147,6 +149,8 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_ every reader of the report, and of the README built from it, that its weights were 4-bit. """ delegate = "XNNPACK with extended ops" if backend == "xnnpack" else "the Vulkan delegate" + # A correction describes the toolchain the file was built with, not the one running now. + version = executorch or toolchain()["executorch"] common = { "qmode": qmode_label, "embedding_quantize": recipe.embedding_quantize, @@ -160,7 +164,7 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_ "int4_codes": None, "label": "fp32 linears, int8 embeddings", "description": ( - f"ExecuTorch {toolchain()['executorch']} `export_llm`: fp32 weights and activations in every " + f"ExecuTorch {version} `export_llm`: fp32 weights and activations in every " f"linear (not quantized), int8 per-channel embeddings, {delegate}, prefill chunk " f"{prefill_chunk}, fp32 KV cache." ), @@ -174,7 +178,7 @@ def recipe_fields(recipe, qmode_label: str, solved: bool, backend: str, prefill_ + (", int4 codes solved with GPTQ" if solved else ", int4 codes rounded to nearest") ), "description": ( - f"ExecuTorch {toolchain()['executorch']} `export_llm`: 8-bit dynamic activations " + f"ExecuTorch {version} `export_llm`: 8-bit dynamic activations " f"and 4-bit weights in groups of {recipe.group_size}, int8 per-channel embeddings, " f"{delegate}, prefill chunk {prefill_chunk}, fp32 KV cache." ), diff --git a/tests/test_correct.py b/tests/test_correct.py new file mode 100644 index 0000000..3edaf2f --- /dev/null +++ b/tests/test_correct.py @@ -0,0 +1,86 @@ +"""pipeline/correct.py: an fp32 report's description and sizing corrected, nothing else, never guessed.""" + +import copy +import dataclasses + +import pytest +from conftest import TOTAL_PARAMS, hf_config + +from pipeline import correct, families, settings, sizing +from pipeline.export_xnnpack import recipe_fields + +MODEL = "HuggingFaceTB/SmolLM2-360M-Instruct" + + +def _published(context=8192): + """A report as the fp32 recipe wrote it before the fix: 8da4w wording and sizing.""" + cfg = settings.load() + arch = families.architecture(hf_config(MODEL), TOTAL_PARAMS[MODEL]) + old = recipe_fields(cfg.xnnpack, "8da4w", False, "xnnpack", 2048, executorch="1.4.0") + old.update({"qmode": "fp32", "label": "fp32-g32, int8 embeddings, int4 codes rounded to nearest"}) + table = [ + { + "context": c, + "device_resident_bytes": sizing.device_resident_bytes(arch, c, cfg.runtime_overhead_bytes), + "fits_device": True, + "export_peak_bytes": 1, + "fits_host": True, + } + for c in (32768, 16384, 8192, 4096, 2048) + ] + report = { + "backend": "xnnpack", + "toolchain": {"executorch": "1.4.0"}, + "source": {"id": MODEL, "sha": "a" * 40}, + "recipe": {"export_llm_qmode": None, "embedding_hqq": False, **old}, + "window": { + "context": context, + "device_resident_bytes": sizing.device_resident_bytes(arch, context, cfg.runtime_overhead_bytes), + "phone_budget_bytes": cfg.device_budget_bytes, + "fits_phone_budget": True, + "table": table, + }, + "estimates": {"pte_bytes_estimate": sizing.pte_bytes_estimate(arch, context)}, + "files": [{"path": "xnnpack/SmolLM2-360M-Instruct-fp32-8k.pte", "bytes": 1499480448, "sha256": "f" * 64}], + } + return report, arch, cfg + + +def test_an_fp32_report_is_corrected_and_only_its_described_fields_change(): + report, arch, cfg = _published() + fixed = correct.corrected(report, arch, cfg, "abc1234", "2026-10-04") + assert fixed["recipe"]["int4_codes"] is None and fixed["recipe"]["group_size"] is None + assert "4-bit" not in fixed["recipe"]["description"] and "ExecuTorch 1.4.0" in fixed["recipe"]["description"] + assert fixed["window"]["device_resident_bytes"] == sizing.device_resident_bytes( + arch, 8192, cfg.runtime_overhead_bytes, "fp32" + ) + assert fixed["estimates"]["pte_bytes_estimate"] == sizing.pte_bytes_estimate(arch, 8192, "fp32") + assert fixed["files"] == report["files"] # the .pte is never touched + assert fixed["corrections"][-1]["by"] == "execupack abc1234" + # Everything outside the listed fields is as it was. + trimmed = copy.deepcopy(fixed) + for key in ("group_size", "int4_codes", "label", "description"): + trimmed["recipe"][key] = report["recipe"][key] + trimmed["window"] = report["window"] + trimmed["estimates"] = report["estimates"] + del trimmed["corrections"] + assert trimmed == report + # Correcting twice changes nothing more. + assert correct.corrected(fixed, arch, cfg, "abc1234", "2026-10-04") is None + + +def test_8da4w_reports_are_left_alone(): + report, arch, cfg = _published() + report["recipe"]["qmode"] = "8da4w-gptq" + assert correct.corrected(report, arch, cfg, "x", "d") is None + + +def test_figures_that_cannot_be_reproduced_are_refused(): + report, arch, cfg = _published() + report["window"]["device_resident_bytes"] += 1 + with pytest.raises(correct.CannotCorrect, match="neither"): + correct.corrected(report, arch, cfg, "x", "d") + report, arch, cfg = _published() + other = dataclasses.replace(cfg, runtime_overhead_bytes=cfg.runtime_overhead_bytes + 1) + with pytest.raises(correct.CannotCorrect): + correct.corrected(report, arch, other, "x", "d")