diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..89b407f --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +# The Namespace runner profile every workflow runs on (runs-on: namespace-profile-execupack). +self-hosted-runner: + labels: + - namespace-profile-execupack diff --git a/.github/workflows/audit-published.yml b/.github/workflows/audit-published.yml index ea195d8..b809447 100644 --- a/.github/workflows/audit-published.yml +++ b/.github/workflows/audit-published.yml @@ -28,7 +28,7 @@ jobs: audit: # x86: the gate runs ExecuTorch's C++ runner through the wheel, which fails on aarch64 # the same way the verify job does (finding 30). - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: namespace-profile-execupack timeout-minutes: 350 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86fefe3..647f792 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,24 @@ permissions: jobs: test: # Lint and unit tests only. - runs-on: blacksmith-2vcpu-ubuntu-2404-arm + runs-on: namespace-profile-execupack steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Runner + # What the Namespace profile really has, read by config/pipeline.yaml's runner_tiers. + run: | + echo "arch $(uname -m), $(nproc) CPUs" + grep -E 'MemTotal|MemAvailable|SwapTotal' /proc/meminfo + df -B1 / /mnt 2>/dev/null || df -B1 / + if sudo -n fallocate -l 1G /exe-expo-swaptest 2>/dev/null && sudo mkswap /exe-expo-swaptest >/dev/null \ + && sudo swapon /exe-expo-swaptest; then + echo "swap file: works"; sudo swapoff /exe-expo-swaptest + else + echo "swap file: refused" + fi + sudo rm -f /exe-expo-swaptest + - name: Read versions id: versions run: grep -E '^[A-Z_]+=' config/versions.env >>"$GITHUB_OUTPUT" diff --git a/.github/workflows/export-mtk.yml b/.github/workflows/export-mtk.yml index c737e42..aee880d 100644 --- a/.github/workflows/export-mtk.yml +++ b/.github/workflows/export-mtk.yml @@ -21,7 +21,7 @@ on: default: "" type: string max_chunks: - description: Maximum chunks to divide the model into (LFM2.5 at 16k built with at most one attention layer per chunk: 8 for the 1.2B, 10 for the 2.6B); empty for mtk.max_chunks in config/pipeline.yaml + description: "Maximum chunks to divide the model into (LFM2.5 at 16k built with at most one attention layer per chunk, 8 for the 1.2B and 10 for the 2.6B); empty for mtk.max_chunks in config/pipeline.yaml" default: "" type: string precision: @@ -50,9 +50,8 @@ concurrency: jobs: targets: - # Hub metadata only, no model download. ARM is fine here precisely because it never - # touches MediaTek's x86-only toolchain; it reads a config.json and does arithmetic. - runs-on: blacksmith-2vcpu-ubuntu-2404-arm + # Hub metadata only, no model download: it reads a config.json and does arithmetic. + runs-on: namespace-profile-execupack env: # The matrix now reads the model's config to size each window's runner, and some of # the watched repos are private, so this job needs the token the export job has. @@ -89,6 +88,8 @@ jobs: export: needs: targets + # An empty matrix is an error to GitHub, so a model no runner can carry stops at targets. + if: ${{ needs.targets.outputs.entries != '[]' }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/export-qnn.yml b/.github/workflows/export-qnn.yml index 1d1d4d4..37a37b1 100644 --- a/.github/workflows/export-qnn.yml +++ b/.github/workflows/export-qnn.yml @@ -39,7 +39,7 @@ concurrency: jobs: targets: # Hub metadata only, no model download. - runs-on: blacksmith-2vcpu-ubuntu-2404-arm + runs-on: namespace-profile-execupack outputs: socs: ${{ steps.socs.outputs.socs }} contexts: ${{ steps.contexts.outputs.contexts }} @@ -87,7 +87,7 @@ jobs: # Stays on x86: the Qualcomm backend ships only in the executorch Linux x86_64 # wheel, and the QAIRT SDK it downloads is x86_64 (this workflow checks for # libQnnHtp.so under an x86_64 path). - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: namespace-profile-execupack timeout-minutes: 350 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/.github/workflows/export-vulkan.yml b/.github/workflows/export-vulkan.yml index bd890f9..a5899e5 100644 --- a/.github/workflows/export-vulkan.yml +++ b/.github/workflows/export-vulkan.yml @@ -35,7 +35,7 @@ concurrency: jobs: targets: # Hub metadata only, no model download: reads config.json and does arithmetic. - runs-on: blacksmith-2vcpu-ubuntu-2404-arm + runs-on: namespace-profile-execupack env: # Sizing reads the model's config, and gated repos (Llama 3.2) need the token. HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -60,16 +60,17 @@ jobs: export: needs: targets + # An empty matrix is an error to GitHub, so a model no runner can carry stops at targets. + if: ${{ needs.targets.outputs.entries != '[]' }} strategy: fail-fast: false matrix: entry: ${{ fromJSON(needs.targets.outputs.entries) }} # Sized per window by the targets job: the export peak grows with the window squared # (every attention layer holds a window x window causal mask), so Qwen3-4B needs ~33 GiB - # at 2k and ~108 GiB at 32k. The old fixed 8vcpu (32 GB + 24 GiB swap) could not carry - # any 32k window of these families but Llama-3.2-1B's. - # Exports stay on x86: the 128 GB tier exists only there (ARM tops out at 96 GB), and the - # runner has no GPU, so the file gets a structural check either way (findings 28, 30). + # at 2k and ~108 GiB at 32k. A window no tier in vulkan.runner_tiers can carry is left + # out; since 2026-10-07 that tier is the 15.7 GiB Namespace profile, with no swap. The + # runner has no GPU, so the file gets a structural check (findings 28, 30). runs-on: ${{ matrix.entry.runner }} timeout-minutes: 350 env: diff --git a/.github/workflows/export-xnnpack.yml b/.github/workflows/export-xnnpack.yml index 13281da..833b70b 100644 --- a/.github/workflows/export-xnnpack.yml +++ b/.github/workflows/export-xnnpack.yml @@ -43,7 +43,7 @@ concurrency: jobs: targets: # Hub metadata only, no model download: reads config.json and does arithmetic. - runs-on: blacksmith-2vcpu-ubuntu-2404-arm + runs-on: namespace-profile-execupack env: # Sizing reads the model's config, and gated repos (Llama 3.2) need the token. HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -67,10 +67,12 @@ jobs: echo "$entries" echo "entries=$entries" >>"$GITHUB_OUTPUT" echo "contexts=$(python3 -c 'import json,sys; print(json.dumps([e["context"] for e in json.loads(sys.argv[1])]))' "$entries")" >>"$GITHUB_OUTPUT" - # The solve holds the fp32 model (16 GB for a 4B) and one layer's Hessians. Every - # solve so far, up to LFM2.5-2.6B, fit the 24 GB size; past 3B it gets the 48 GB one. + # The solve holds the fp32 model (16 GB for a 4B) and one layer's Hessians. On + # Blacksmith every solve up to LFM2.5-2.6B fit 23.4 GiB and past 3B took 48 GB. One + # Namespace profile (15.7 GiB) now runs it; the models that would not fit it have no + # window that fits an export either, so their solve is skipped with the empty matrix. params=$(python3 -c 'import sys; from pipeline import hub; print(hub.fetch(sys.argv[1], sys.argv[2]).total_params or 0)' "$MODEL_ID" "$REVISION") - if [ "$params" -gt 3000000000 ]; then solve=blacksmith-16vcpu-ubuntu-2404-arm; else solve=blacksmith-8vcpu-ubuntu-2404-arm; fi + solve=namespace-profile-execupack echo "solve runner: $solve ($params parameters)" echo "solve_runner=$solve" >>"$GITHUB_OUTPUT" @@ -79,9 +81,8 @@ jobs: # the KV cache and the masks, never the weights. Every window's export reuses this # artifact, which is why it is worth a job of its own. needs: targets - if: ${{ inputs.gptq }} - # ARM like the export: the solve works on the eager PyTorch module and never loads the - # C++ runner, which is the only thing aarch64 cannot do (finding 30). + # Nothing to solve for when no window fits a runner (an empty matrix). + if: ${{ inputs.gptq && needs.targets.outputs.entries != '[]' }} # Sized on memory: it holds the fp32 model and one layer's Hessians (targets job). runs-on: ${{ needs.targets.outputs.solve_runner }} timeout-minutes: 350 @@ -117,15 +118,16 @@ jobs: export: needs: [targets, solve] - if: ${{ always() && needs.targets.result == 'success' && needs.solve.result != 'failure' }} + # An empty matrix is an error to GitHub, so a model no runner can carry stops at targets. + if: ${{ always() && needs.targets.result == 'success' && needs.targets.outputs.entries != '[]' && needs.solve.result != 'failure' }} strategy: fail-fast: false matrix: entry: ${{ fromJSON(needs.targets.outputs.entries) }} # Sized per window by the targets job: every attention layer holds a window x window - # causal mask, so the fixed 24 GB ARM runner this used to be could not carry 32k for any - # Qwen3, Qwen2.5, Llama-3.2-3B or SmolLM2 export. x86 tiers, for the 128 GB one (ARM tops - # out at 96); the export is arch-clean and the smoke test stays in its own x86 job. + # causal mask, so the need grows with the window squared, and a window no tier in + # xnnpack.runner_tiers can carry is left out. Since 2026-10-07 that tier is the 15.7 GiB + # Namespace profile (x86, no swap). The smoke test stays in its own job (verify). runs-on: ${{ matrix.entry.runner }} timeout-minutes: 350 env: @@ -185,7 +187,7 @@ jobs: verify: # The smoke test, and only then the publish. ExecuTorch's runner raises IndexError: stoi - # on aarch64 (finding 30), so this is the one job that has to be x86; it loads the .pte + # on aarch64 (finding 30), so this job has to be x86, as every job is now; it loads the .pte # the export built and asks the same question through the same C++ runner the app uses. needs: [targets, export] # Not `needs.export.result == 'success'`: the export matrix is fail-fast: false so that @@ -195,12 +197,12 @@ jobs: # verified and never published because of it. Each verify job already stands down on # its own when its window uploaded no artifact, which is the per-window check this # should have relied on all along. - if: ${{ always() && needs.targets.result == 'success' }} + if: ${{ always() && needs.targets.result == 'success' && needs.targets.outputs.entries != '[]' }} strategy: fail-fast: false matrix: context: ${{ fromJSON(needs.targets.outputs.contexts) }} - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: namespace-profile-execupack timeout-minutes: 60 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/.github/workflows/probe-runner.yml b/.github/workflows/probe-runner.yml index a0cdc63..2bb27a2 100644 --- a/.github/workflows/probe-runner.yml +++ b/.github/workflows/probe-runner.yml @@ -11,7 +11,7 @@ on: type: string contexts: description: JSON list of windows, one job each - default: "[2048, 16384]" + default: "[2048, 8192]" type: string permissions: @@ -23,15 +23,11 @@ jobs: fail-fast: false matrix: context: ${{ fromJSON(inputs.contexts) }} - # Sized on memory, not cores: 32 GB, because the causal masks grow with the window - # squared and have killed a 32k export before (docs/research findings 2 and 28). The - # CPU knee is not yet measured. - # Exports stay on x86 until the runner works on aarch64. The toolchain is arm64-clean and - # the export itself succeeded there, but the wheel's C++ TextLLMRunner raises - # `IndexError: stoi` on ARM, and that runner is the smoke test: without it a published - # file drops from "generated text and answered" to "the methods exist", which is the - # standard the QNN and MediaTek programs were withdrawn for (findings 28, 30). - runs-on: blacksmith-8vcpu-ubuntu-2404 + # The causal masks grow with the window squared and have killed a 32k export before + # (docs/research findings 2 and 28). x86, because the wheel's C++ TextLLMRunner raises + # `IndexError: stoi` on aarch64 (finding 30). The Namespace profile has no swap, so the + # swap asked for below is not added there. + runs-on: namespace-profile-execupack timeout-minutes: 120 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -52,7 +48,17 @@ jobs: swap-gib: "24" - name: Export at a forced window - run: python -m pipeline export-xnnpack "$MODEL_ID" --context "$CONTEXT" --out "$WORK_DIR/out" --work "$WORK_DIR/work" + # Exit 4 is the gate refusing a window this runner cannot hold (16k and up for + # Qwen3-0.6B on the Namespace profile): a measurement in itself, not a failure. + run: | + set +e + python -m pipeline export-xnnpack "$MODEL_ID" --context "$CONTEXT" --out "$WORK_DIR/out" --work "$WORK_DIR/work" + code=$? + if [ "$code" -eq 4 ]; then + echo "::notice::Window $CONTEXT skipped: it does not fit this runner." + exit 0 + fi + exit "$code" - name: Summary if: always() diff --git a/.github/workflows/watch-hf.yml b/.github/workflows/watch-hf.yml index 2f194db..d455934 100644 --- a/.github/workflows/watch-hf.yml +++ b/.github/workflows/watch-hf.yml @@ -40,7 +40,7 @@ concurrency: jobs: watch: # Hub metadata only, no model download. - runs-on: blacksmith-2vcpu-ubuntu-2404-arm + runs-on: namespace-profile-execupack timeout-minutes: 30 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/CLAUDE.md b/CLAUDE.md index 89ed801..241f375 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co exe-expo exports small dense open-weight LLMs (≤4B, no MoE) from Hugging Face to ExecuTorch `.pte` files for the [openweights](https://github.com/alpharomercoma/openweights) Android app, running entirely on hosted CI runners -(Blacksmith, Ubuntu ARM for everything except the smoke test: the solve and the export run on `blacksmith-8vcpu-ubuntu-2404-arm`, and a `verify` job on x86 runs ExecuTorch's C++ runner, which fails on aarch64, before anything publishes. See finding 30.) `docs/PLAN.md` holds the decisions, measured costs, phase status and known +(Namespace since 2026-10-07: every job runs on `namespace-profile-execupack`, 4 x86 vCPUs, 15.7 GiB of RAM and no swap, which the container refuses; a `verify` job runs ExecuTorch's C++ runner before anything publishes. The sizing tiers in `config/pipeline.yaml` are that one profile, so a window that does not fit its RAM is skipped (finding 40); a larger Namespace profile added there as a second tier brings the wider windows back.) `docs/PLAN.md` holds the decisions, measured costs, phase status and known limits. Read it before changing sizing, backends or naming, and update it when a decision or measurement changes. ## Commands diff --git a/README.md b/README.md index e52a2d6..79ee885 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,8 @@ Exports small open-weight LLMs from Hugging Face to ExecuTorch `.pte` files for the [openweights](https://github.com/alpharomercoma/openweights) Android app, on -[Blacksmith](https://www.blacksmith.sh)'s hosted CI runners: ARM for the GPTQ solve and the -XNNPACK export, x86 for the runner check before anything publishes, and x86 at 8, 16 or 32 -vCPUs for MediaTek, sized per context window. Design and decisions: [docs/PLAN.md](docs/PLAN.md). +[Namespace](https://namespace.so) runners (`namespace-profile-execupack`: 4 x86 vCPUs, 16 GB, +no swap), with every window checked against that memory before a job is started. Design and decisions: [docs/PLAN.md](docs/PLAN.md). What has shipped to [`experimentalmachines`](https://huggingface.co/experimentalmachines) on Hugging Face, and what is built but not published: @@ -12,8 +11,8 @@ Hugging Face, and what is built but not published: | Backend | Status | |---|---| | XNNPACK (CPU) | **Published**: LFM2.5 1.2B and 2.6B (and their heretic finetunes), Qwen3, Qwen2.5, Llama 3.2, SmolLM2, at every window from 2k to 32k the runner could build. Each window is now compared with its fp32 model before it publishes. Twelve models were published before that gate existed and are being re-checked by **Audit published**; the two audited so far, SmolLM2-135M and 360M, score 0.19 and 0.22 on its prompts against about 0.58 for their fp32 models | -| MediaTek NeuroPilot (MT6991) | **Published**: LFM2.5 1.2B and 2.6B (and their heretic finetunes) at MediaTek's default 512-token window. The sizing now puts 2k to 32k within the 32-vCPU tier; a 4k export has been built and answered correctly on a Dimensity 9400, and nothing above 512 is published yet. Checked structurally on the runner, which has no NPU | -| Vulkan (GPU) | Workflow in place, same models and recipe as XNNPACK through ExecuTorch's Vulkan delegate, checked structurally (no GPU on the runner). Nothing published yet | +| MediaTek NeuroPilot (MT6991) | **Published**: LFM2.5 1.2B and 2.6B (and their heretic finetunes) from 512 up to 8k, and the heretic finetunes at 16k; Qwen3, Llama 3.2 and SmolLM2 at the windows that finished before the 2026-10-04 cancel. Built on Blacksmith runners of up to 128 GB; the Namespace profile used since 2026-10-07 cannot build LFM2.5 at any window (finding 40). Checked structurally on the runner, which has no NPU | +| Vulkan (GPU) | **Published**: Qwen3, Qwen2.5, Llama 3.2 and SmolLM2 at 2k to 32k, same recipe as XNNPACK through ExecuTorch's Vulkan delegate, checked structurally on the runner (no GPU) and run through both apps on a Mali and two Adreno GPUs (finding 38) | | Qualcomm QNN (SM8750; SM8650 optional) | Workflow in place for the checkpoints in ExecuTorch 1.4.0's Qualcomm registry (Qwen3, Qwen2.5 base, Gemma 3 1B, SmolLM2 135M, SmolLM3 3B, Llama 3.2). Nothing published yet | | Samsung Exynos (ENN) | Waiting for an LLM path in ExecuTorch (1.4.0 ships the delegate with CNN examples only) | | HF watcher (auto-dispatch) | Run by hand: its hourly schedule has been off since 2026-09-19. Dispatches XNNPACK for every model first, then Vulkan, then QNN, then MediaTek; state on the `state` branch | @@ -79,8 +78,9 @@ stages are 87-93% of it: - The first MediaTek export (Qwen3-0.6B, MT6991, 512-token cache) took 1,695 s: calibration 736 s (43%), NeuroPilot lowering 472 s (28%), over 4 chunks. Its limit was calibration memory, which held every prompt at once and pinned every window at 512. Holding one prompt at - a time moved the rest to disk, so the runner for each window is now picked by RAM and disk - together, and the sizing puts 2k to 32k within the 32-vCPU tier (4k built so far). + a time moved the rest to disk, so the runner for each window is picked by RAM and disk + together. On the Blacksmith tiers that put 2k to 32k within the 32-vCPU one; on the one + Namespace profile used since 2026-10-07 (15.7 GiB, no swap) no LFM2.5 window fits (finding 40). ## Locally diff --git a/config/pipeline.yaml b/config/pipeline.yaml index ba937d9..df02ca8 100644 --- a/config/pipeline.yaml +++ b/config/pipeline.yaml @@ -76,21 +76,13 @@ export: # built on two hosted runners differs in the embedding's low bits and small models decode # different text. Affine is deterministic; leave this false to keep builds reproducible. embedding_hqq: false - # The same x86 tiers as vulkan.runner_tiers (see there for the measured RAM), picked per + # The same tier as vulkan.runner_tiers (see there for what was measured), picked per # window by export_xnnpack.pick_runner on sizing.host_need_bytes(..., "xnnpack"). runner_tiers: - - label: blacksmith-8vcpu-ubuntu-2404 - ram_bytes: 33229729792 - disk_bytes: 160000000000 - swap_gib: 24 - - label: blacksmith-16vcpu-ubuntu-2404 - ram_bytes: 66577858560 - disk_bytes: 750000000000 - swap_gib: 64 - - label: blacksmith-32vcpu-ubuntu-2404 - ram_bytes: 121766014976 - disk_bytes: 1500000000000 - swap_gib: 96 + - label: namespace-profile-execupack + ram_bytes: 16833269760 + disk_bytes: 101427666944 + swap_gib: 0 vulkan: # The GPU delegate runs the same torchao 8da4w linears (backends/vulkan/vulkan-quantization); # the file is otherwise the XNNPACK recipe with backend.vulkan in export_llm. @@ -98,29 +90,23 @@ export: group_size: 32 embedding_quantize: "8,0" embedding_hqq: false - # x86 Blacksmith runners, smallest first; export_xnnpack.pick_runner gives each window - # the first that holds sizing.host_need_bytes in RAM, and when none can, the one with the - # most RAM that holds it with swap. The cost is the window, not the model: every - # attention layer holds a window x window mask, so Qwen3-4B needs ~22 GiB at 2k and ~134 - # GiB at 32k, and one fixed size either wastes the small windows or kills the wide ones. - # x86 rather than ARM for the largest tier (ARM tops out at 96 GB). + # The runners, smallest first; export_xnnpack.pick_runner gives each window the first that + # holds sizing.host_need_bytes in RAM, and when none can, the one with the most RAM that + # holds it with swap. The cost is the window, not the model: every attention layer holds a + # window x window mask, so Qwen3-4B needs ~22 GiB at 2k and ~134 GiB at 32k. # - # ram_bytes is the smallest MemTotal the runners reported in fifty Vulkan exports - # (2026-10-04), not the nominal size: the 32vcpu tier is sold as 128 GB and reported - # 113.4 to 124.2 GiB, and 8vcpu and 16vcpu reported 30.9 and 62.0 GiB every time. + # Since 2026-10-07 there is one: the Namespace profile namespace-profile-execupack, 4 x86 + # vCPUs. Its own CI run measured MemTotal 16,438,740 kB, a 101 GB overlay disk shared by / + # and /mnt, and no swap, which the container does not allow (swapon: Operation not + # permitted), so swap_gib is 0 and a window needs to fit in RAM. A window no tier can carry + # is left out of the matrix and recorded as skipped; a larger Namespace profile added here + # as a second tier brings the wider windows back. The Blacksmith tiers this replaced (30.9, + # 62.0 and 113.4 GiB, with 24 to 96 GiB of swap) are in docs/PLAN.md. runner_tiers: - - label: blacksmith-8vcpu-ubuntu-2404 - ram_bytes: 33229729792 - disk_bytes: 160000000000 - swap_gib: 24 - - label: blacksmith-16vcpu-ubuntu-2404 - ram_bytes: 66577858560 - disk_bytes: 750000000000 - swap_gib: 64 - - label: blacksmith-32vcpu-ubuntu-2404 - ram_bytes: 121766014976 - disk_bytes: 1500000000000 - swap_gib: 96 + - label: namespace-profile-execupack + ram_bytes: 16833269760 + disk_bytes: 101427666944 + swap_gib: 0 qnn: # HTP context binaries are compiled per chip and load only on that chip. The benchmark # devices (docs/PLAN.md, "Devices"): Snapdragon 8 Elite (Galaxy S25 Ultra) = SM8750. @@ -150,7 +136,7 @@ export: # every prompt's steps at once, each with a full fp32 KV cache at the window, and 2048 # exhausted a 16.8 GB runner and took it down. The patched scripts pass # writer_batch_size=1 so one prompt is resident instead of all nine, which takes the - # corpus size out of the peak and puts 2k to 32k within a blacksmith-32vcpu runner. + # corpus size out of the peak and put 2k to 32k within the 128 GB runners used until 2026-10-07. # This stays at MediaTek's default because it is the floor of the matrix; the windows # actually built come from context_tiers through the workflow's contexts input. prompt_tokens: 128 @@ -170,32 +156,24 @@ export: # window that does not fit is skipped outright rather than calibrated on too little. # Each export report still records prompts_used, so a reader can see what was used. min_calibration_prompts: 4 - # x86 Blacksmith runners, smallest first; export_mtk.pick_runner takes the first that - # carries a window. x86 because mtk_converter is a cp310 x86 wheel and mtk_neuron is - # tagged linux_x86_64, so no ARM tier can run the toolchain at all, whatever its size. + # The runners, smallest first; export_mtk.pick_runner takes the first that carries a + # window. x86 because mtk_converter is a cp310 x86 wheel and mtk_neuron is tagged + # linux_x86_64, so no ARM machine can run the toolchain at all, whatever its size. # # For qwen.py both numbers bind. Memory is one prompt's calibration steps; disk is every - # prompt's, because writer_batch_size=1 moved them to the Arrow cache. The 1.2B at 16k is - # the case that shows why disk is listed: it has the RAM for an 8vcpu runner and not the - # disk. swap_gib is what the workflow asks prepare-host.sh for on that tier, sized to - # leave the cache room on the same filesystem. + # prompt's, because writer_batch_size=1 moved them to the Arrow cache. lfm2.py calibrates + # in one streaming pass that keeps no step, so time is what binds there: max_window is the + # largest window a tier is given, so a calibration lands on cores that finish it inside the + # job limit. # - # lfm2.py calibrates in one streaming pass that keeps no step, so neither grows with the - # window much and time is what binds: max_window is the largest window a tier is given, - # so a 32k calibration, which runs a quarter of a million tokens through the model twice, - # lands on the cores that finish it inside the job limit. + # One tier since 2026-10-07, the Namespace profile (vulkan.runner_tiers says what was + # measured): 15.7 GiB, no swap, 4 vCPUs. calibration_bytes puts a 1.2B LFM2.5 build at + # about 17.4 GiB (15 bytes a parameter, measured on 4-chunk builds), so none is placed; the + # 8-chunk 16k build peaked at 13.4 GiB, so a refit for more chunks may bring it back. + # max_window 2048 because 4 vCPUs are half the 8 that were given at most 4096. runner_tiers: - - label: blacksmith-8vcpu-ubuntu-2404 - ram_bytes: 32000000000 - disk_bytes: 160000000000 - swap_gib: 24 - max_window: 4096 - - label: blacksmith-16vcpu-ubuntu-2404 - ram_bytes: 64000000000 - disk_bytes: 750000000000 - swap_gib: 64 - max_window: 16384 - - label: blacksmith-32vcpu-ubuntu-2404 - ram_bytes: 128000000000 - disk_bytes: 1500000000000 - swap_gib: 96 + - label: namespace-profile-execupack + ram_bytes: 16833269760 + disk_bytes: 101427666944 + swap_gib: 0 + max_window: 2048 diff --git a/docs/PLAN.md b/docs/PLAN.md index 3459aa0..601dddb 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -14,7 +14,7 @@ for it, Samsung Exynos (ENN). iOS is deferred. | Watched orgs | `Qwen`, `google`, `meta-llama`, `HuggingFaceTB`, each for its own families | | Trigger | Hourly watcher; eligible models are dispatched in stages, XNNPACK for every model first, then Vulkan, then Qualcomm, then MediaTek (a stage waits until the previous one has nothing queued or running) | | First run | Seeds state without exporting; existing models are backfilled by manual dispatch | -| Runners | Blacksmith, one workflow run per backend and one job per window. Ubuntu ARM throughout except the smoke test: solve and export on `blacksmith-8vcpu-ubuntu-2404-arm`, orchestration and CI on the 2 vCPU ARM size, and a `verify` job on x86 that runs the C++ runner (which fails on aarch64) and publishes only once the file has answered (finding 30) | +| Runners | Namespace since 2026-10-07: one workflow run per backend and one job per window, every job on `namespace-profile-execupack` (4 x86 vCPUs, 15.7 GiB, no swap), and a `verify` job that runs the C++ runner and publishes only once the file has answered (finding 30). A window that does not fit the profile's RAM is left out of the matrix (finding 40). Until then: Blacksmith, ARM for orchestration, solve and export, x86 tiers of 30.9, 62.0 and 113.4 GiB with 24 to 96 GiB of swap for the wide windows | | Chips | The benchmark devices ("Devices" below): QNN SM8750 (Snapdragon 8 Elite), MediaTek MT6989 (Dimensity 9300+); SM8650 and MT6991 can be added in `config/pipeline.yaml` | | Outputs | Hugging Face Hub, GitHub Releases (files ≤ 2 GiB), Actions artifacts | | HF layout | One repo per model, backend folders; NPU exports published even though the app can't load them yet | diff --git a/docs/research/README.md b/docs/research/README.md index b439627..0f98541 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -65,3 +65,4 @@ Findings from building and running this pipeline, each kept with its evidence. C | 37 | 2026-10-04 | MediaTek | Two faults in MediaTek's stock Qwen and Llama paths, both visible in the fp32 graph before any quantization, measured against Hugging Face over 301 tokens of one feed. RoPE: `get_master_rot_emb` builds the table at base 10000 and neither `QwenConfig` nor `LlamaConfig` keeps `rope_theta` (Qwen 1e6, Llama 3.2 5e5, SmolLM2 1e5), and Llama 3's frequency scaling is not implemented; Qwen3-0.6B was at KL 0.082, top-1 93.0%, perplexity 2.609 against 2.549. Mask: the runner's additive -100 lets Qwen2.5's unnormalised scores leak through masked positions, KL 0.013, top-1 98.0%, perplexity 2.277 against 2.196 on Qwen2.5-0.5B-Instruct, exact at -10000; Qwen3 (QK norm), Llama 3.2 and SmolLM2 are exact at -100. Also: `writer_batch_size=1`, which `calibration_bytes` assumes, was only ever in the old lfm2.py; `qwen.py` buffered every prompt. | measured | `qwen_vs_hf.py` harness (MediaTek chunk classes on CPU with mtk_converter stubbed, as in finding 36's) | `mediatek-rope-theta.patch` keeps `rope_theta` and implements llama3 scaling: Qwen3-0.6B, Llama-3.2-1B and SmolLM2-135M then match Hugging Face exactly (KL 0, top-1 100%). The calibration patch adds `writer_batch_size=1` to qwen.py and llama.py. Qwen2.5 is refused for MediaTek until the mask is changed in the runner and graph. | | 38 | 2026-10-04 | Vulkan | The Vulkan exports run, through both apps, on a Mali and two Adreno GPUs, and whether they beat the XNNPACK file depends on the GPU. Qwen3-0.6B 2k, ExecuTorch 1.5.1 runtime (`executorch-android-vulkan`): openweights' nine engine device tests pass on a Poco X8 Pro Max (MT6991, Mali-G925), a Snapdragon 8 Gen 3 (SM8650, Adreno 750) and a Snapdragon 8 Elite Gen 5 (SM8850); ExecuServe serves the GPU file on all three, and passes its OpenAI 16/16, edge-case 28/28 and Anthropic 8/8 suites on the Poco. On the Poco's Mali-G925 the CPU file decodes 2.2 to 3.9 times faster at both measured prompt lengths (52.1 tok/s against 18.1 and 15.7 through `llama_main`; through ExecuServe 56 to 59 against 15 to 16 after a 27-token prompt and 30 against 13 to 14 after a 702-token one), and the GPU only reads a long prompt faster (371 to 432 tok/s against 325 to 335; engine 402 to 427 against 274 to 279). On the SM8850, through ExecuServe's run history, the GPU reads a 702-token prompt at 901 to 1,180 tok/s against the CPU's 746 to 771 and decodes after it at 61 to 64 against 50 to 51, but decodes after a 27-token prompt at 81 to 93 against 123 to 128; the engine's 937-token matrix agrees (prefill 968 to 1,117 against 548 to 556, decode 51 to 55 against 38 to 39). Resident memory for the same GPU file was 222 to 268 MB on the Adrenos and about 2.2 GB on the Mali, so it is a property of the driver, not the model. | measured | openweights [`docs/research/vulkan-on-device.md`](https://github.com/ExperimentalMachines/openweights/blob/main/docs/research/vulkan-on-device.md) and its raw logs, runs CSV and Codex reviews in [`tools/eval/results/vulkan-2026-10-04/`](https://github.com/ExperimentalMachines/openweights/blob/main/tools/eval/results/vulkan-2026-10-04/); ExecuServe `docs/results/2026-10-04-vulkan.md` | Vulkan stays in the export matrix: it is correct everywhere tried and the better file on a recent Adreno for long prompts. The apps, not the exporter, choose per phone; on Mali the CPU file is the one to use, which the apps do not yet recommend. Only Qwen3-0.6B was measured; hybrid families on 1.5.1's Vulkan delegate are untried. | | 39 | 2026-10-04 | MediaTek | At 16k, LFM2.5 built once no chunk held more than one attention layer; a 4-chunk build with two in some chunks did not. LFM2.5-1.2B-Instruct (attention at layers 2, 5, 8, 10, 12, 14 of 16) at 16k in 4 chunks lost MediaTek's local compiler service while compiling chunk 2, after calibration had finished; its heretic finetune, the same architecture, in 8 chunks (two layers, at most one attention layer each) built and was published: 3,235 s, peak 14,361,923,584 B in use on a 16 vCPU runner. LFM2.5-2.6B (attention at 2, 5, 9, 13, 17, 21, 24, 27 of 30) needs 10 chunks for one attention layer each, and its heretic finetune built at 16k that way: 5,891 s, peak 26,554,892,288 B. Two successes and one failure, so one attention layer per chunk is a working heuristic for these architectures; that it is necessary, and why the compiler service died, are not shown. 32k was started for all three LFM2.5 models (8 and 10 chunks) and cancelled mid-calibration for budget. | measured + inference | run 35994412279 (4 chunks, failed at chunk 2), run 37171790063 (8 chunks, published), run 37176914465 (2.6B-heretic, 10 chunks, 16k published, 32k cancelled); `mtk/mt6991/export-report-16k.json` in `experimentalmachines/LFM2.5-1.2B-Instruct-heretic-ExecuTorch` and `…/LFM2.5-2.6B-heretic-ExecuTorch` | Dispatch 16k with `max_chunks` at the smallest count that leaves one attention layer per chunk (8 for the 1.2B, 10 for the 2.6B); `mtk.max_chunks` stays 4 for the windows that build at it. The successful builds peaked at 13.4 and 24.7 GiB; what killed the failed run's compiler service is not known. LFM2.5-2.6B (base) at 16k and every 32k remain unbuilt. | +| 40 | 2026-10-07 | Runner | The Namespace profile `namespace-profile-execupack` has 4 x86_64 CPUs, MemTotal 16,438,740 kB (15.7 GiB), a 101 GB overlay disk shared by / and /mnt, and no swap: `swapon` on a swap file is refused ("Operation not permitted"), so the 24 to 96 GiB the Blacksmith jobs added cannot be had. Against `sizing.host_need_bytes` and `export_mtk.calibration_bytes` that leaves: Vulkan up to 8k for the 0.1B to 1.2B models (Qwen3-0.6B, Qwen2.5-0.5B, SmolLM2, Llama-3.2-1B) and up to 16k for LFM2.5-1.2B, 2k-4k for Qwen2.5-1.5B, Qwen3-1.7B and LFM2.5-2.6B, nothing at 3B and above; XNNPACK up to 8k for the sub-1B models, 2k-4k for Llama-3.2-1B and LFM2.5-1.2B, nothing from 1.5B up; MediaTek Qwen3-0.6B at 512 only, Llama-3.2-1B to 2k, SmolLM2-135M to 4k, and no LFM2.5 at any window by the current estimate (about 17.4 GiB for the 1.2B). Of the windows built before, every 32k window and every 16k except LFM2.5-1.2B's on Vulkan now fall outside it, as do the MediaTek 16k builds of finding 39. That last is the estimate's call, not a measured limit: `calibration_bytes` puts the 1.2B at about 17.4 GiB from the 14.4 to 15.7 bytes per parameter measured on 4-chunk builds, while the 8-chunk 16k build peaked at 13.4 GiB, which this RAM would hold; refitting the estimate for more chunks could bring the 1.2B back. | measured + estimate | CI run 37606922033 (the runner step); `tests/test_namespace_runner.py` pins the tier and the windows; the coverage is the sizing model's, not exports run on the profile | `config/pipeline.yaml` has the one tier with swap_gib 0; `prepare-host.sh` carries on without swap where it is refused; a model no window fits stops at the targets job instead of failing an empty matrix. The wider windows need a larger Namespace profile added as a second tier. QNN has no tier: its export peaked at 15.6 GB of RSS for Qwen3-0.6B (finding 8), at the edge of this profile. | diff --git a/scripts/ci/prepare-host.sh b/scripts/ci/prepare-host.sh index a6337d9..438a58a 100644 --- a/scripts/ci/prepare-host.sh +++ b/scripts/ci/prepare-host.sh @@ -57,12 +57,22 @@ fi sudo swapoff -a || true sudo rm -f /mnt/swapfile /swapfile /mnt/exe-expo.swap /exe-expo.swap swap_file="${swap_base%/}/exe-expo.swap" -sudo fallocate -l "${SWAP_GIB}G" "$swap_file" -sudo chmod 600 "$swap_file" -sudo mkswap "$swap_file" >/dev/null -sudo swapon "$swap_file" +# Namespace's runners are containers that refuse swapon ("Operation not permitted", measured +# 2026-10-07 on namespace-profile-execupack). There the export has its RAM and nothing more; +# the job's own gate reads SwapTotal from /proc/meminfo (exporting.host_budget), so it budgets +# for what is really there rather than for the swap asked for. +if [ "$SWAP_GIB" -eq 0 ]; then + swap_note="no swap: none asked for" +elif sudo fallocate -l "${SWAP_GIB}G" "$swap_file" && sudo chmod 600 "$swap_file" \ + && sudo mkswap "$swap_file" >/dev/null && sudo swapon "$swap_file" 2>/dev/null; then + swap_note="swap in $swap_file" +else + sudo rm -f "$swap_file" + swap_note="no swap: this runner does not allow it" + echo "::warning::This runner does not allow swap; the export has $(free -g | awk '/^Mem:/{print $2}') GiB of RAM and no more." +fi -show "host after cleanup (work in $work_dir, swap in $swap_file)" +show "host after cleanup (work in $work_dir, $swap_note)" if [ -n "${GITHUB_ENV:-}" ]; then echo "WORK_DIR=$work_dir" >>"$GITHUB_ENV" diff --git a/tests/conftest.py b/tests/conftest.py index cd60a0a..2a840e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ import pytest from pipeline.hub import SourceModel +from pipeline.settings import RunnerTier FIXTURES = Path(__file__).parent / "fixtures" @@ -61,3 +62,21 @@ def make_source(model_id: str, config: dict | None = None, **overrides) -> Sourc @pytest.fixture def source(): return make_source + + +# A three-size fleet for testing the runner choice itself: the x86 runners execupack used until +# 2026-10-07, as their export reports measured them (30.9, 62.0 and 113.4 GiB of RAM). The +# configured tiers are one Namespace profile now (test_namespace_runner.py), but the choice +# has to keep working the day a larger profile is added beside it. +FLEET = ( + RunnerTier("small", 33_229_729_792, 160_000_000_000, 24), + RunnerTier("medium", 66_577_858_560, 750_000_000_000, 64), + RunnerTier("large", 121_766_014_976, 1_500_000_000_000, 96), +) +# The same fleet as config/pipeline.yaml's mtk tiers had it: nominal sizes, and the largest +# window each was given for a streaming (lfm2.py) calibration. +MTK_FLEET = ( + RunnerTier("small", 32_000_000_000, 160_000_000_000, 24, 4096), + RunnerTier("medium", 64_000_000_000, 750_000_000_000, 64, 16384), + RunnerTier("large", 128_000_000_000, 1_500_000_000_000, 96), +) diff --git a/tests/test_mtk.py b/tests/test_mtk.py index 6ce79ad..a8fc36a 100644 --- a/tests/test_mtk.py +++ b/tests/test_mtk.py @@ -3,11 +3,13 @@ import subprocess import pytest -from conftest import TOTAL_PARAMS, hf_config, load_json +from conftest import MTK_FLEET, TOTAL_PARAMS, hf_config, load_json from pipeline import export_mtk, families, manifest, naming, publish, settings CFG = settings.load() +# The runner choice is tested on a three-size fleet (conftest); the configured tier is one. +FLEET = dataclasses.replace(CFG.mtk, runner_tiers=MTK_FLEET) def flag(command, name): @@ -227,8 +229,8 @@ def test_calibration_memory_estimate_covers_the_measured_peak_and_refuses_what_d assert at_512 < 29_592_731_648 / 3 # LFM2.5-1.2B and 2.6B are what this matrix exports: 16 and 30 layers, 8 KV heads, head - # dim 64. Every window from 2k to 32k has to fit a blacksmith-32vcpu runner, 128 GB with - # the 64 GiB swap the workflow adds, or the window cannot be built there. + # dim 64. Every window from 2k to 32k had to fit the 128 GB runner used until 2026-10-07, + # with the 64 GiB swap the workflow added, or the window could not be built there. budget = 128_000_000_000 + 64 * 1024**3 - 1_000_000_000 for layers, model_params in ((16, 1_170_340_608), (30, 2_600_000_000)): lfm = { @@ -257,7 +259,7 @@ def test_calibration_disk_is_what_the_runner_tier_has_to_carry(): cache = export_mtk.calibration_disk_bytes(lfm, at_32k, 9) assert cache == 9 * 10 * 65_536 * 32_768 # 193 GB - # A blacksmith-8vcpu runner has 160 GB and cannot hold it; 16vcpu has 750 GB and can. + # The small fleet runner's 160 GB disk cannot hold it; the medium one's 750 GB can. assert cache + export_mtk.DISK_MARGIN_BYTES > 160 * 10**9 assert cache + export_mtk.DISK_MARGIN_BYTES < 750 * 10**9 @@ -277,26 +279,26 @@ def test_the_runner_is_sized_by_the_window_and_both_limits_bind(): lfm26 = dict(lfm12, num_hidden_layers=30) def pick(config, window, params): - tier = export_mtk.pick_runner(config, dataclasses.replace(CFG.mtk, cache_size=window), 9, params) + tier = export_mtk.pick_runner(config, dataclasses.replace(FLEET, cache_size=window), 9, params) return tier.label if tier else None # The window sets the size, not the model: the same model wants three different tiers. - assert pick(lfm12, 2048, 1_170_340_608) == "blacksmith-8vcpu-ubuntu-2404" - assert pick(lfm12, 8192, 1_170_340_608) == "blacksmith-16vcpu-ubuntu-2404" - assert pick(lfm12, 32768, 1_170_340_608) == "blacksmith-32vcpu-ubuntu-2404" + assert pick(lfm12, 2048, 1_170_340_608) == "small" + assert pick(lfm12, 8192, 1_170_340_608) == "medium" + assert pick(lfm12, 32768, 1_170_340_608) == "large" # And the model matters at a fixed window, because both terms carry its shape. - assert pick(lfm26, 8192, 2_600_000_000) == "blacksmith-32vcpu-ubuntu-2404" + assert pick(lfm26, 8192, 2_600_000_000) == "large" # Tiers are tried smallest first, so a window that fits the smallest never gets a # bigger one: that is the saving this exists for. - smallest = CFG.mtk.runner_tiers[0] + smallest = FLEET.runner_tiers[0] assert pick(lfm12, 2048, 1_170_340_608) == smallest.label # Nothing is handed a runner that cannot hold it. The 2.6B at 32k needs about 157 GiB # against the largest tier's 128 GB, so it is the one window placed on swap, and it is # still placed rather than dropped. - biggest = CFG.mtk.runner_tiers[-1] + biggest = FLEET.runner_tiers[-1] assert pick(lfm26, 32768, 2_600_000_000) == biggest.label at_32k = dataclasses.replace(CFG.mtk, cache_size=32768) need = export_mtk.calibration_bytes(lfm26, at_32k, 9, 2_600_000_000) @@ -440,18 +442,18 @@ def test_streaming_calibration_keeps_nothing_on_disk_and_barely_grows_with_the_w def test_streaming_windows_go_to_the_cores_that_finish_them(): def pick(config, window, params): - recipe = dataclasses.replace(CFG.mtk, cache_size=window) + recipe = dataclasses.replace(FLEET, cache_size=window) tier = export_mtk.pick_runner(config, recipe, 23, params, streaming=True) return tier.label if tier else None - assert pick(LFM12, 2048, 1_170_340_608) == "blacksmith-8vcpu-ubuntu-2404" - assert pick(LFM12, 4096, 1_170_340_608) == "blacksmith-8vcpu-ubuntu-2404" - assert pick(LFM12, 16384, 1_170_340_608) == "blacksmith-16vcpu-ubuntu-2404" - assert pick(LFM12, 32768, 1_170_340_608) == "blacksmith-32vcpu-ubuntu-2404" + assert pick(LFM12, 2048, 1_170_340_608) == "small" + assert pick(LFM12, 4096, 1_170_340_608) == "small" + assert pick(LFM12, 16384, 1_170_340_608) == "medium" + assert pick(LFM12, 32768, 1_170_340_608) == "large" # The 2.6B's lowering needs more than the smallest tier holds in RAM at any window, and # its 32k window is a time question, not a memory one: it no longer needs swap. - assert pick(LFM26, 2048, 2_697_198_592) == "blacksmith-16vcpu-ubuntu-2404" - biggest = CFG.mtk.runner_tiers[-1] + assert pick(LFM26, 2048, 2_697_198_592) == "medium" + biggest = FLEET.runner_tiers[-1] at_32k = dataclasses.replace(CFG.mtk, cache_size=32768) assert export_mtk.calibration_bytes(LFM26, at_32k, 23, 2_697_198_592, streaming=True) < biggest.ram_bytes diff --git a/tests/test_namespace_runner.py b/tests/test_namespace_runner.py new file mode 100644 index 0000000..9cb491f --- /dev/null +++ b/tests/test_namespace_runner.py @@ -0,0 +1,86 @@ +"""The configured runner: one Namespace profile, and what it can and cannot build. + +namespace-profile-execupack replaced the Blacksmith tiers on 2026-10-07. Its own CI run +measured 4 x86_64 CPUs, MemTotal 16,438,740 kB, a 101 GB overlay disk shared by / and /mnt, +and no swap, which the container refuses (swapon: Operation not permitted). These tests pin +those figures in config/pipeline.yaml and what follows from them, so a change to either is a +decision someone makes rather than a matrix that quietly shrinks or a job that runs out of +memory. +""" + +import dataclasses + +import pytest +from conftest import TOTAL_PARAMS, hf_config + +from pipeline import export_mtk, export_xnnpack, families, settings + +CFG = settings.load() +PROFILE = "namespace-profile-execupack" +MEM_TOTAL_BYTES = 16_438_740 * 1024 +WINDOWS = (2048, 4096, 8192, 16384, 32768) + + +def arch(model_id): + return families.architecture(hf_config(model_id), TOTAL_PARAMS[model_id]) + + +def windows(model_id, backend): + tiers = getattr(CFG, backend).runner_tiers + return [w for w in WINDOWS if export_xnnpack.pick_runner(arch(model_id), w, tiers, backend)] + + +@pytest.mark.parametrize("backend", ["xnnpack", "vulkan", "mtk"]) +def test_every_recipe_runs_on_the_namespace_profile_as_measured(backend): + (tier,) = getattr(CFG, backend).runner_tiers + assert tier.label == PROFILE + assert tier.ram_bytes == MEM_TOTAL_BYTES + # No swap: the container does not allow it, so nothing may be sized on swap. + assert tier.swap_gib == 0 + + +def test_no_workflow_runs_anywhere_else(): + for workflow in (settings.ROOT / ".github/workflows").glob("*.yml"): + text = workflow.read_text(encoding="utf-8") + assert "blacksmith" not in text, workflow.name + for line in text.splitlines(): + if line.strip().startswith("runs-on:") and "${{" not in line: + assert line.split("runs-on:")[1].strip() == PROFILE, f"{workflow.name}: {line.strip()}" + + +@pytest.mark.parametrize( + ("model_id", "backend", "expected"), + [ + # 15.7 GiB less the reserve, against host_need_bytes: the masks grow with the window + # squared, so the small models keep up to 8k and lose 16k and 32k. + ("Qwen/Qwen3-0.6B", "vulkan", [2048, 4096, 8192]), + ("Qwen/Qwen3-0.6B", "xnnpack", [2048, 4096, 8192]), + ("HuggingFaceTB/SmolLM2-360M-Instruct", "vulkan", [2048, 4096, 8192]), + ("meta-llama/Llama-3.2-1B-Instruct", "vulkan", [2048, 4096, 8192]), + ("meta-llama/Llama-3.2-1B-Instruct", "xnnpack", [2048, 4096]), + # XNNPACK's 1.7x headroom on the fp32 weights puts the 1.5B to 4B models out of reach. + ("Qwen/Qwen2.5-1.5B-Instruct", "vulkan", [2048, 4096]), + ("Qwen/Qwen2.5-1.5B-Instruct", "xnnpack", []), + ("Qwen/Qwen3-1.7B", "xnnpack", []), + ("Qwen/Qwen3-4B", "vulkan", []), + ("Qwen/Qwen3-4B", "xnnpack", []), + ], +) +def test_what_the_profile_can_export(model_id, backend, expected): + assert windows(model_id, backend) == expected + + +def test_mediatek_lfm2_does_not_fit_at_any_window(): + # By the current estimate: calibration_bytes takes 14.4 to 15.7 bytes a parameter, measured + # on 4-chunk builds, about 17.4 GiB for the 1.2B. The 8-chunk 16k build peaked at 13.4 GiB + # (finding 39), so this pins the estimate's answer, not a measured limit (finding 40). + lfm = { + "num_hidden_layers": 16, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "hidden_size": 2048, + } + for window in (512, 2048, 4096): + recipe = dataclasses.replace(CFG.mtk, cache_size=window) + assert export_mtk.pick_runner(lfm, recipe, 23, 1_170_340_608, streaming=True) is None diff --git a/tests/test_runner_tiers.py b/tests/test_runner_tiers.py index 239e7bd..3023ae8 100644 --- a/tests/test_runner_tiers.py +++ b/tests/test_runner_tiers.py @@ -1,11 +1,12 @@ -"""Per-window runner choice for the export_llm backends (export-vulkan.yml's matrix).""" +"""Per-window runner choice for the export_llm backends (export-vulkan.yml's matrix), on a +three-size fleet; what the configured tier itself carries is test_namespace_runner.py.""" import pytest -from conftest import TOTAL_PARAMS, hf_config +from conftest import FLEET, TOTAL_PARAMS, hf_config from pipeline import export_xnnpack, families, settings, sizing -TIERS = settings.load().vulkan.runner_tiers +TIERS = FLEET SMALL, MEDIUM, LARGE = (t.label for t in TIERS) GIB = 2**30 @@ -20,7 +21,9 @@ def pick(model_id, window): def test_tiers_are_smallest_first(): - assert [t.ram_bytes for t in TIERS] == sorted(t.ram_bytes for t in TIERS) + cfg = settings.load() + for tiers in (TIERS, cfg.xnnpack.runner_tiers, cfg.vulkan.runner_tiers, cfg.mtk.runner_tiers): + assert [t.ram_bytes for t in tiers] == sorted(t.ram_bytes for t in tiers) @pytest.mark.parametrize( @@ -61,7 +64,7 @@ def test_the_vulkan_need_does_not_inherit_the_xnnpack_headroom_at_small_windows( def test_when_nothing_holds_it_in_ram_the_most_ram_wins(): - # Qwen3-4B at 32k needs more than any tier's RAM: the 32vcpu tier with swap, never a + # Qwen3-4B at 32k needs more than any tier's RAM: the largest tier with swap, never a # smaller tier paging more of it. qwen = arch("Qwen/Qwen3-4B") need = sizing.host_need_bytes(qwen, 32768, "vulkan")