diff --git a/.github/workflows/export-vulkan.yml b/.github/workflows/export-vulkan.yml deleted file mode 100644 index 18a20f5..0000000 --- a/.github/workflows/export-vulkan.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: Export Vulkan -run-name: "Vulkan: ${{ inputs.model_id }}@${{ inputs.revision }}${{ inputs.contexts && format(' ctx {0}', inputs.contexts) || '' }}" - -on: - workflow_dispatch: - inputs: - model_id: - description: Hugging Face model id, e.g. Qwen/Qwen3-1.7B - required: true - type: string - revision: - description: Branch, tag or commit of the source repo - default: main - type: string - contexts: - description: JSON list of context windows in tokens, one job each; empty for every tier in config/pipeline.yaml - default: "" - type: string - publish_hf: - description: Publish to the Hugging Face org in config/pipeline.yaml - default: true - type: boolean - publish_release: - description: Attach the files to a GitHub release - default: true - type: boolean - -permissions: - contents: write # GitHub releases - -concurrency: - group: export-vulkan-${{ inputs.model_id }} - cancel-in-progress: false - -jobs: - targets: - # Hub metadata only, no model download: reads config.json and does arithmetic. - runs-on: ubuntu-latest - env: - # Sizing reads the model's config, and gated repos (Llama 3.2) need the token. - HF_TOKEN: ${{ secrets.HF_TOKEN }} - outputs: - entries: ${{ steps.contexts.outputs.entries }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/setup-light - - id: contexts - env: - CONTEXTS: ${{ inputs.contexts }} - MODEL_ID: ${{ inputs.model_id }} - REVISION: ${{ inputs.revision }} - # One entry per window, each with the smallest runner that holds it - # (pipeline.export_xnnpack.pick_runner over vulkan.runner_tiers). The export job gates - # on the same sizing.host_need_bytes, so the two cannot disagree about a window; one - # no tier can carry is left out here rather than started on a runner it would kill. - run: | - entries=$(python3 -m pipeline export-matrix "$MODEL_ID" --revision "$REVISION" --backend vulkan --contexts "$CONTEXTS") - echo "$entries" - echo "entries=$entries" >>"$GITHUB_OUTPUT" - - export: - needs: targets - # An empty matrix is an error to GitHub, so a model no runner can carry stops at targets. - if: ${{ needs.targets.outputs.entries != '[]' }} - strategy: - fail-fast: false - matrix: - entry: ${{ fromJSON(needs.targets.outputs.entries) }} - # The export runs on Modal, in a Sandbox with the memory this window needs (the targets - # job's memory_mib: sizing.host_need_bytes plus modal.memory_margin): the peak grows with - # the window squared, from ~5 GiB at 2k to ~134 GiB for Qwen3-4B at 32k. This job only - # sends it, waits, and publishes what comes back. Modal has no GPU either, so the file - # gets a structural check (findings 28, 30). - runs-on: ubuntu-latest - timeout-minutes: 350 - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} - MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - HF_HUB_DISABLE_PROGRESS_BARS: "1" - MODEL_ID: ${{ inputs.model_id }} - REVISION: ${{ inputs.revision }} - CONTEXT: ${{ matrix.entry.context }} - MEMORY_MIB: ${{ matrix.entry.memory_mib }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - uses: ./.github/actions/setup-modal - - # Exit 4 means this window cannot be built (the export's own gate): recorded as a skip, - # so the other windows' jobs stand on their own. - - name: Export and check on Modal - run: | - echo "ARTIFACT=vulkan-${MODEL_ID//\//--}-${CONTEXT}" >>"$GITHUB_ENV" - set +e - python -m pipeline modal-run --memory-mib "$MEMORY_MIB" --requirements requirements/export-xnnpack.txt \ - --get "$WORK_DIR/out" \ - --script 'python -m pipeline export-vulkan "$MODEL_ID" --revision "$REVISION" --context "$CONTEXT" --out "$OUT" --work "$WORK"' - rc=$? - set -e - if [ "$rc" -eq 4 ]; then - echo "SKIPPED=1" >>"$GITHUB_ENV" - echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens cannot be exported" - exit 0 - fi - exit "$rc" - - - name: Summary - if: always() && env.SKIPPED != '1' - run: | - if compgen -G "$WORK_DIR/out/vulkan/export-report*.json" >/dev/null; then - python -m pipeline summary "$WORK_DIR/out" --backend vulkan >>"$GITHUB_STEP_SUMMARY" - fi - - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() && env.SKIPPED != '1' - with: - name: ${{ env.ARTIFACT }} - path: ${{ env.WORK_DIR }}/out - retention-days: 14 - if-no-files-found: warn - - - name: Publish to Hugging Face - if: inputs.publish_hf && env.SKIPPED != '1' - run: | - if [ -z "$HF_TOKEN" ]; then - echo "::error::Set the HF_TOKEN repository secret (write access to the output org)" - exit 1 - fi - python -m pipeline publish-hf "$WORK_DIR/out" --backend vulkan - - - name: Publish GitHub release - if: inputs.publish_release && env.SKIPPED != '1' - env: - GH_TOKEN: ${{ github.token }} - run: python -m pipeline publish-release "$WORK_DIR/out" --backend vulkan diff --git a/.github/workflows/export-xnnpack.yml b/.github/workflows/export-xnnpack.yml index 7c71147..6de30f3 100644 --- a/.github/workflows/export-xnnpack.yml +++ b/.github/workflows/export-xnnpack.yml @@ -24,6 +24,10 @@ on: description: "Recipe: empty for the config's 8da4w, or fp32 (linears not quantized; the solve only writes the gate's fp32 reference)" default: "" type: string + vulkan: + description: Also build each window's Vulkan (GPU) file from the same solved codes, gated through that window's XNNPACK file (GPTQ 8da4w runs of models Vulkan can take) + default: true + type: boolean publish_hf: description: Publish to the Hugging Face org in config/pipeline.yaml default: true @@ -58,11 +62,21 @@ jobs: CONTEXTS: ${{ inputs.contexts }} MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} + VULKAN: ${{ inputs.vulkan }} + GPTQ: ${{ inputs.gptq }} + QMODE: ${{ inputs.qmode }} # One entry per window with the memory its Modal Sandbox asks for (export-matrix over # xnnpack.runner_tiers: sizing.host_need_bytes plus modal.memory_margin). A window past - # what Modal allows is left out here. + # what Modal allows is left out here. With --with-vulkan each entry also says whether + # its job builds the Vulkan file (the model can take Vulkan and the window fits Modal), + # and asks for whichever of the two exports needs more. Never for a rounded or fp32 run: + # a GPU file is built only from solved codes, gated through its XNNPACK twin. run: | - entries=$(python3 -m pipeline export-matrix "$MODEL_ID" --revision "$REVISION" --backend xnnpack --contexts "$CONTEXTS") + WITH_VULKAN="" + if [ "$VULKAN" = "true" ] && [ "$GPTQ" = "true" ] && { [ -z "$QMODE" ] || [ "$QMODE" = "8da4w" ]; }; then + WITH_VULKAN="--with-vulkan" + fi + entries=$(python3 -m pipeline export-matrix "$MODEL_ID" --revision "$REVISION" --backend xnnpack --contexts "$CONTEXTS" $WITH_VULKAN) echo "$entries" echo "entries=$entries" >>"$GITHUB_OUTPUT" # The solve holds the fp32 model (16 GB for a 4B) and one layer's Hessians: about 12 @@ -127,6 +141,16 @@ jobs: # test through ExecuTorch's C++ runner, then the decision gate against the fp32 model's own # answers, and only then does this job publish. They used to be two jobs because the # runner fails on aarch64 (finding 30) and the export ran on ARM; on Modal both are x86. + # + # Then, for a window targets marked (export-matrix --with-vulkan: a GPTQ 8da4w run of a + # model Vulkan can take), the Vulkan file, in a Sandbox of its own (vulkan_memory_mib): the + # same solved codes in the same torchao linears, with the XNNPACK report just published as + # its --gate-proxy. The host has no Vulkan runtime, so that verdict is the one it carries, + # and export_xnnpack.check_gate_proxy refuses one built any other way. The CPU file is + # published before the GPU Sandbox starts, so a GPU failure or timeout never costs it; + # publishing it also withdraws, in the same commit, a Vulkan window whose gate was the file + # it replaces (publish.orphaned_gpu_window), so the Hub never holds a GPU file citing a + # gate that is not there. A Vulkan failure fails the job at the end. runs-on: ubuntu-latest timeout-minutes: 350 env: @@ -140,6 +164,8 @@ jobs: GPTQ: ${{ inputs.gptq }} CONTEXT: ${{ matrix.entry.context }} MEMORY_MIB: ${{ matrix.entry.memory_mib }} + VULKAN: ${{ matrix.entry.vulkan && 'true' || 'false' }} + VULKAN_MEMORY_MIB: ${{ matrix.entry.vulkan_memory_mib }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -156,6 +182,7 @@ jobs: # job had: a solved run with no fp32 reference refuses to publish rather than falling # through to an unguarded one. - name: Export, smoke-test and gate on Modal + id: export run: | echo "ARTIFACT=xnnpack-${MODEL_ID//\//--}-${CONTEXT}" >>"$GITHUB_ENV" puts=() @@ -205,6 +232,7 @@ jobs: if-no-files-found: warn - name: Publish to Hugging Face + id: publish_xnnpack if: inputs.publish_hf && env.SKIPPED != '1' run: | if [ -z "$HF_TOKEN" ]; then @@ -214,7 +242,62 @@ jobs: python -m pipeline publish-hf "$WORK_DIR/out" --backend xnnpack - name: Publish GitHub release + id: release_xnnpack if: inputs.publish_release && env.SKIPPED != '1' env: GH_TOKEN: ${{ github.token }} run: python -m pipeline publish-release "$WORK_DIR/out" --backend xnnpack + + # Its own Sandbox and its own clock: whatever happens here, the XNNPACK file above is + # already out. Exit 4 is the Vulkan export's own size gate (no file for this window). + - name: Build the Vulkan file on Modal + if: ${{ !cancelled() && env.VULKAN == 'true' && env.SKIPPED != '1' && steps.export.outcome == 'success' && steps.publish_xnnpack.outcome != 'failure' }} + timeout-minutes: 130 + run: | + proxy=$(ls "$WORK_DIR"/out/xnnpack/export-report-*.json | head -1) + set +e + python -m pipeline modal-run --memory-mib "$VULKAN_MEMORY_MIB" --timeout-minutes 120 \ + --requirements requirements/export-xnnpack.txt \ + --put "$WORK_DIR/in/codes.pt" --put "$proxy" --get "$WORK_DIR/vulkan-out" \ + --script 'python -m pipeline export-vulkan "$MODEL_ID" --revision "$REVISION" --context "$CONTEXT" \ + --out "$OUT" --work "$WORK/vulkan" --codes "$IN/codes.pt" --gate-proxy "$(ls "$IN"/export-report-*.json)"' + rc=$? + set -e + if [ "$rc" -eq 0 ]; then + echo "VULKAN_BUILT=1" >>"$GITHUB_ENV" + python -m pipeline summary "$WORK_DIR/vulkan-out" --backend vulkan >>"$GITHUB_STEP_SUMMARY" + elif [ "$rc" -eq 4 ]; then + echo "::warning title=Vulkan window skipped::$MODEL_ID at $CONTEXT tokens cannot be exported for Vulkan" + else + echo "VULKAN_FAILED=1" >>"$GITHUB_ENV" + echo "::error title=Vulkan export failed::exit $rc; the XNNPACK file is published, this job fails at the end" + fi + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ !cancelled() && (env.VULKAN_BUILT == '1' || env.VULKAN_FAILED == '1') }} + with: + name: vulkan-${{ env.ARTIFACT }} + path: ${{ env.WORK_DIR }}/vulkan-out + retention-days: 14 + if-no-files-found: warn + + # Only once its twin is on the Hub: a GPU file is never published citing a gate that + # is not there to read. + - name: Publish the Vulkan file to Hugging Face + id: publish_vulkan + if: ${{ !cancelled() && inputs.publish_hf && env.VULKAN_BUILT == '1' && steps.publish_xnnpack.outcome == 'success' }} + run: python -m pipeline publish-hf "$WORK_DIR/vulkan-out" --backend vulkan + + # Likewise only beside its twin's release, and, when the Hub is published to, only once the + # GPU file is there too (a skipped step is not a success). + - name: Publish the Vulkan GitHub release + if: ${{ !cancelled() && inputs.publish_release && env.VULKAN_BUILT == '1' && steps.release_xnnpack.outcome == 'success' && (!inputs.publish_hf || steps.publish_vulkan.outcome == 'success') }} + env: + GH_TOKEN: ${{ github.token }} + run: python -m pipeline publish-release "$WORK_DIR/vulkan-out" --backend vulkan + + - name: Fail on a Vulkan export that did not build + if: ${{ !cancelled() && env.VULKAN_FAILED == '1' }} + run: | + echo "::error::the Vulkan export failed after the XNNPACK file was published; see the Vulkan step" + exit 1 diff --git a/CLAUDE.md b/CLAUDE.md index 6bdc90f..99cb318 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,16 +39,20 @@ there and regenerate; never hand-edit the SVGs. `docs/brand/README.md` has the u - **`watch-hf.yml`** (hourly) → `watch.py`: lists each watched org's newest repos, checks every unseen one (by name first with no network, then by config), and runs `gh workflow run` for each backend's export workflow. State is - `seen.json` on the orphan **`state` branch**. Dispatch is staged (`watch.STAGES`: XNNPACK for every model, then Vulkan, then QNN, then - MediaTek; the next stage starts when the previous has nothing queued or running) and idempotent (a run in flight for - the model counts as dispatched); `requeue` puts cancelled dispatches back. The first run only seeds the state; existing models are exported via the - `backfill` input. `watch.WORKFLOWS` maps backend → workflow. + `seen.json` on the orphan **`state` branch**. Dispatch is staged (`watch.STAGES`: XNNPACK, with Vulkan in the same + run, for every model, then QNN, then MediaTek; the next stage starts when the previous has nothing queued or running) and + idempotent (a run in flight for the model counts as dispatched); `requeue` puts cancelled dispatches back. The first run + only seeds the state; existing models are exported via the `backfill` input. `watch.WORKFLOWS` maps backend → + workflow, and `watch.BUILT_WITH` names a backend built in another's run (Vulkan in XNNPACK's). - **`solve` job in `export-xnnpack.yml`** (on Modal) → `solve.py`: GPTQ for the int4 codes, once per model because the codes are per linear and the window changes only the KV cache and the masks. Calibration is `calibration.py`: committed prompts in the model's own chat template, each continued by the fp32 model's own greedy reply. The codes reach every window's export as an artifact; `export_with_codes.py` injects them into torchao's tensors before lowering and also carries the LFM2 state fix (`lfm2_state.py`), which the solve applies too. -- **`export-xnnpack.yml`** / **`export-vulkan.yml`** (each window in a Modal Sandbox via `remote.py`) → `export_xnnpack.py` (`backend="xnnpack"|"vulkan"`, same recipe, XNNPACK or Vulkan delegate; Vulkan gets the structural check): `hub.fetch` → `eligibility.evaluate` → `sizing.choose_context` → +- **`export-xnnpack.yml`** (each window in a Modal Sandbox via `remote.py`) → `export_xnnpack.py` (`backend="xnnpack"`, + then in the same Sandbox `backend="vulkan"` from the same codes with the XNNPACK report as `--gate-proxy`, for GPTQ + 8da4w runs of models Vulkan can take; there is no Vulkan workflow of its own, since one could only solve again or round + to nearest; Vulkan gets the structural check): `hub.fetch` → `eligibility.evaluate` → `sizing.choose_context` → download → `convert.py` (HF safetensors → ExecuTorch checkpoint layout) → generated `params.json` + `export_llm` YAML → subprocess `executorch.extension.llm.export.export_llm` → `smoke.py` (greedy generation through the wheel's `TextLLMRunner`, the same C++ runner the app uses) → `export-report-.json` + `config.json`. diff --git a/docs/PLAN.md b/docs/PLAN.md index 138a9ae..789b92c 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -140,14 +140,36 @@ kernels and the runner no GPU, so the check is structural (`smoke.structural_che gate must also have passed or been accepted. The proxy is recorded in the Vulkan report, and `publish` refuses a GPTQ Vulkan file without one. Why: round to nearest kept none of Phi-4-mini's 8 graded tool calls (mean 0.05 against fp32's 0.66, measured on an XNNPACK - file with the same quantization). The `export-vulkan` workflow does not pass `--codes` - yet, so CI's Vulkan files are still round to nearest. + file with the same quantization). +- **Built in the XNNPACK window job** (2026-11-09): `export-xnnpack.yml` builds each window's + Vulkan file right after that window's XNNPACK file has passed its gate and been published, + from the same `codes.pt`, with that report as `--gate-proxy`, in a Modal Sandbox of its own + (`vulkan_memory_mib`, 120 minutes), and publishes it only once its twin is on the Hub. A + Vulkan failure or timeout therefore never costs the CPU file, and fails the job at the end. + `export-matrix --with-vulkan` marks the windows that get one (the model can take Vulkan and + the window can be sized). Only GPTQ 8da4w runs build one: a rounded or fp32 run gets no GPU + file. Publishing an XNNPACK window withdraws, in the same Hub commit, a Vulkan window whose + gate proxy names, by path and sha256, a twin this commit removes or replaces with other bytes + (`publish.orphaned_gpu_window`, judged on the tree the commit produces), so a failed or + skipped GPU rebuild leaves no GPU file citing a gate that is gone. A GPTQ Vulkan publish in turn requires the head it + commits onto to hold the very XNNPACK file its proxy names (`publish.require_twin_on_hub`, + re-checked on every retry); `publish-hf --allow-unpublished-twin` is the explicit way past it + for a twin deliberately not published, as SmolLM2-360M's is. GitHub releases follow the Hub: + a GPU release needs its twin on the Hub at release time, and re-releasing an XNNPACK window + under other bytes deletes the GPU release of that window whose proxy named the old ones. + Limits: releases are per source revision, so an older revision's GPU release stays beside its + own XNNPACK release while the Hub moves on; and the check-then-release is not atomic against a + second publisher acting by hand at the same moment (CI runs one model at a time). `export-vulkan.yml` is gone, and the watcher has no Vulkan stage (`watch.BUILT_WITH`): + a workflow of its own could only solve again, giving the CPU and GPU files different codes, + or round to nearest, which is what every Vulkan file it built was. - **Runners, per window** (2026-10-04): the export peak grows with the window squared (the same per-layer causal masks as XNNPACK), so one fixed 8vcpu runner could not carry any 32k - window of Qwen3, Qwen2.5 or Llama-3.2-3B. The `targets` job runs `export-matrix`, which - gives each window the smallest x86 tier in `vulkan.runner_tiers` that holds + window of Qwen3, Qwen2.5 or Llama-3.2-3B. The `targets` job ran `export-matrix`, which + gave each window the smallest x86 tier in `vulkan.runner_tiers` that holds `sizing.host_need_bytes(..., "vulkan")` in RAM, or, when none can, the tier with the most - RAM that holds it with swap. The job gates on the same number. + RAM that holds it with swap. Since 2026-11-09 the same choice is made by the XNNPACK + workflow's `export-matrix --with-vulkan`, as each window's `vulkan_memory_mib` for its + Vulkan Sandbox (the tier is Modal's); a window it cannot size gets no Vulkan file. - **Vulkan memory, measured** (50 exports: Qwen3 0.6B/1.7B/4B/4B-2507, Qwen2.5 0.5B/1.5B/ Math-1.5B/3B, Llama 3.2 1B/3B, each at 2k-32k, all published): against the XNNPACK estimate the peak was 0.76-0.97x up to 8k, 0.99-1.55x at 16k and 1.54-2.13x at 32k, so the @@ -450,14 +472,30 @@ G5, Dimensity 9300+, Exynos 2500; GSM8K, RetrievalQA, IFEval, PopQA, BFCL, Fresh that cannot fit is refused. That guard was in place and did not fire for Qwen3-1.7B at 32k, because the estimate it gates on was itself too low; see the headroom factor above. -- **Eleven of the sixteen published repositories are round to nearest, not GPTQ**: both - Llama-3.2, all four Qwen2.5, Qwen3-0.6B, Qwen3-4B, Qwen3-4B-Instruct-2507 and both - SmolLM2. Only the four LFM2.5 repositories and Qwen3-1.7B carry solved codes. Round to - nearest is what took LFM2.5's tool-call recall from 49 percent to 10. `python -m pipeline - audit ` gates a published file against its fp32 source without rebuilding it, so - this can be measured without a solve per model; its first run found - SmolLM2-135M-Instruct-ExecuTorch failing, at 0.190 against fp32's 0.583 over the four - rows fp32 is confident on, with the quiet rows untouched. +- **No round-to-nearest CPU or GPU file is published** (2026-11-09). The XNNPACK folders had + been re-exported with GPTQ by October; every Vulkan folder was still round to nearest, because + the Vulkan export had never taken codes. On 2026-11-09 the 13 repos with a Vulkan folder were + rebuilt by hand on EC2 from main c01c60e: per model one solve (Qwen3-4B-Thinking-2507 reused + the solve its XNNPACK files came from), and per window an XNNPACK twin from those codes + (gated, published) and a Vulkan file with the twin as its gate proxy (published). Before each publish, a byte check read every int4 linear out of the `.pte` + through the delegate's own fully-connected (XNNPACK) or `linear_dq8ca_q4gsw` (Vulkan) ops in + execution order and required it to equal `codes.pt`, codes and scales, and the report to name + that `codes.pt` by sha256. Afterwards every Hub file's sha256 was matched to a byte-checked + file, every Vulkan proxy to the published twin and its verdict, and every XNNPACK verdict to + the model's own `gate.json`. The tools (`codesmatch.py`, `ptecodes.py`, `verify_hub.py`) + are not in this repository yet. Exceptions, decided by hand: + - Qwen3-0.6B: int4 does not keep its tool decision (research finding 43), so its XNNPACK + files are fp32 linears (gated: 4 of 4, 0.618 against fp32's 0.619) and its Vulkan folder + was withdrawn (repo commit d60f655). + - SmolLM2-360M: XNNPACK stays fp32 (its 1.4.0 GPTQ file failed the gate); the GPTQ Vulkan + files are gated through an XNNPACK twin that passed (2 of 2 graded rows, 0.555 against + 0.570) but was not published, so their proxy names a file that is not on the Hub. + From then on CI builds Vulkan files the same way, in the XNNPACK window job (Vulkan section). +- **Gates before 2026-10-31 did not measure Qwen3's decision.** A gate run before `6ee6b36` + (reasoning switched off) graded `` at 0.9999 on every row, so every Qwen3 file built + before then passed whatever its weights. `python -m pipeline audit Qwen/Qwen3-0.6B --context + 2048` on the published October GPTQ file: 0 of 4 calls kept, 0.086 against fp32's 0.619. + The 2026-11-09 rebuilds replaced every such Qwen3 file with one gated the corrected way. - The watcher checks each repo once. A model whose weights change after it was seen (a new commit on `main`) is not re-exported automatically: most upstream commits touch diff --git a/docs/research/README.md b/docs/research/README.md index e58953e..30fe41d 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -68,3 +68,6 @@ Findings from building and running this pipeline, each kept with its evidence. C | 40 | 2026-10-07 | Runner | The Namespace profile `namespace-profile-execupack` has 4 x86_64 CPUs, MemTotal 16,438,740 kB (15.7 GiB), a 101 GB overlay disk shared by / and /mnt, and no swap: `swapon` on a swap file is refused ("Operation not permitted"), so the 24 to 96 GiB the Blacksmith jobs added cannot be had. Against `sizing.host_need_bytes` and `export_mtk.calibration_bytes` that leaves: Vulkan up to 8k for the 0.1B to 1.2B models (Qwen3-0.6B, Qwen2.5-0.5B, SmolLM2, Llama-3.2-1B) and up to 16k for LFM2.5-1.2B, 2k-4k for Qwen2.5-1.5B, Qwen3-1.7B and LFM2.5-2.6B, nothing at 3B and above; XNNPACK up to 8k for the sub-1B models, 2k-4k for Llama-3.2-1B and LFM2.5-1.2B, nothing from 1.5B up; MediaTek Qwen3-0.6B at 512 only, Llama-3.2-1B to 2k, SmolLM2-135M to 4k, and no LFM2.5 at any window by the current estimate (about 17.4 GiB for the 1.2B). Of the windows built before, every 32k window and every 16k except LFM2.5-1.2B's on Vulkan now fall outside it, as do the MediaTek 16k builds of finding 39. That last is the estimate's call, not a measured limit: `calibration_bytes` puts the 1.2B at about 17.4 GiB from the 14.4 to 15.7 bytes per parameter measured on 4-chunk builds, while the 8-chunk 16k build peaked at 13.4 GiB, which this RAM would hold; refitting the estimate for more chunks could bring the 1.2B back. | measured + estimate | CI run 37606922033 (the runner step); `tests/test_namespace_runner.py` pins the tier and the windows; the coverage is the sizing model's, not exports run on the profile | `config/pipeline.yaml` has the one tier with swap_gib 0; `prepare-host.sh` carries on without swap where it is refused; a model no window fits stops at the targets job instead of failing an empty matrix. The wider windows need a larger Namespace profile added as a second tier. QNN has no tier: its export peaked at 15.6 GB of RSS for Qwen3-0.6B (finding 8), at the edge of this profile. | | 41 | 2026-10-07 | Runner | Modal runs the XNNPACK and Vulkan exports and the solve. A Sandbox may ask for at most 344064 MiB (336 GiB) and 64 physical cores: larger requests are refused at creation with InvalidError ("memory request must be <= 344064 MiB", "CPU request must be <= 64000 mCPU"), so every window ever built fits, Qwen3-4B at 32k (about 134 GiB) included. Inside a Sandbox /proc/meminfo shows the host's memory (680 GiB to 1.3 TiB), not the request, so the export's own memory gate passes there and the request computed from `sizing.host_need_bytes` is what sizes the run; nproc does reflect the request. SmolLM2-135M at 2k: the Vulkan file came out byte-identical to the published one (sha256 0de5a249...), peaking at 1.9 GiB of a 6.1 GiB request; the XNNPACK file exported and passed the smoke test ("The capital of France is Paris") in the same Sandbox. Both were slower than on Blacksmith: 361 s against 145 s for Vulkan and 1,308 s against 348 s for XNNPACK, though the published files were built with ExecuTorch 1.4.0 and these with 1.5.1, so the runner is not the only difference. | measured | `pipeline/remote.py` runs from this workstation, 2026-10-07 (the sandbox limits probed by request; the two exports with their export reports); the published reports in `experimentalmachines/SmolLM2-135M-Instruct-ExecuTorch` | xnnpack and vulkan runner_tiers are one `modal` tier at Modal's ceiling, each window's memory_mib is its need times modal.memory_margin, and the XNNPACK export, smoke test and decision gate share one Sandbox, so the separate verify job is gone. Per-export time is the open cost question: modal.cpu is 8 cores, and whether more cores or a 1.5.1-to-1.4.0 difference explains the gap is not measured. | | 42 | 2026-10-31 | XNNPACK | LiquidAI's LFM3-1.7B (`Lfm3ForCausalLM`, remote code only: transformers has no LFM3 and ExecuTorch 1.5.1 none either, the newest release on 2026-10-31) cannot be described by ExecuTorch 1.5.1's transformer: of 24 layers 13 are width-7 gated short convolutions (`ShortConvBlock` hard-codes 3), 6 attend globally with a zero-tail partial RoPE (the first quarter of each head rotated at theta 1e6, frequencies over the full head dimension) and 5 attend over a 256-token window with full RoPE at theta 1e4 (one RoPE table per model upstream; `sliding_window` and `local_rope_theta` are declared in `ModelArgs` and never read), every attention output is gated per head by `sigmoid(gate_proj(x))` (hidden to 16; upstream's `use_attn_o_gate` is per element), and each operator and feed-forward is followed by its own RMSNorm (`post_attention_norm` is declared and never read). `pipeline/lfm3_model.py` builds it from ExecuTorch's parts and is applied in every process that builds the model, as the LFM2 state fix is. The windowed layers keep a ring of window + prefill chunk slots on `update_cache_with_indices` and a masked `custom_sdpa` (upstream's `RingKVCache` holds 2 x window and asserts a chunk fits, which a 2,048-token prefill does not), so no layer holds a full-window cache it cannot use and no window-squared mask exists. On the real checkpoint (ac2586a3) against the checkpoint's own `modeling_lfm3.py`, fp32: without a cache, identical logits over a 559-token app prompt and a 17-token one (max difference 0, top-1 100%); with the cache and export_llm's custom-op transforms, 96-token prefill chunks (the ring wraps), 40 decode steps and a second prompt at position zero, max logit difference 1.8e-4, top-1 100%, KL under 4e-7. The remote config does not load under transformers 5.17.0 (`standardize_rope_params` looks up a RoPE block for every layer type, `conv` included: KeyError); 5.22.1 loads it. | measured + source | `lfm3_check.py` harness (HF fp32 vs the module, no cache and cached); `examples/models/llama/{llama_transformer,model_args,attention}.py` and `examples/models/lfm2/short_conv.py` at v1.5.1 | XNNPACK only (Vulkan has no short-convolution kernel; neither vendor path has a definition). The app has no LFM3 chat template, so the family is exported ahead of it (`naming.AHEAD_OF_APP`) and the model card says so; `-Base` stays unpublished under the chat-only rule, LFM3-VL and the audio models are multimodal. The first published reports counted the 5 windowed layers at a full-window KV cache (1,476,395,008 B at 32k); since eba9368 `sizing` counts them as their fixed ring (852,492,288 B at 32k, 47,185,920 B of it the rings), and `correct-reports` rewrote the five published reports, `config.json` and README in Hub commit fd07e07 without touching a `.pte`. The decision gate first failed all three GPTQ windows on quiet rows: it reads one token, and LFM3's template opens a `` block in the generation prompt, so that token is the first word of the reasoning (fp32 'The' at 0.52 to 0.73 against 'We'), which the 4-bit file tipped the other way on two of four rows without calling anything. Qwen3's published gates had the same blind spot (`` at 0.9999 on every row, `measures_tool_calling` false). With `enable_thinking=False` the prompt ends in ``, and fp32's first token is `` at 1.000 on all 12 search rows and the answer on the quiet ones (0.98, 0.99, 0.95; 'Who wrote Pride and Prejudice?' undecided at 0.27); the 32k GPTQ file then keeps 12 of 12 calls at 0.9991 against 0.9998 and 3 of 4 quiet rows, `measures_tool_calling` true. `gate.prompts` now renders with the reasoning switched off for every family; templates without the switch are unchanged, and calibration still renders at the template's default. ExecuTorch 1.6.0-rc1 (tagged 2026-10-30, wheel on PyTorch's test channel, torch 2.15.0) has no LFM3 model either, though its `ShortConv` takes a width: under it the same module matches Hugging Face (identical logits without a cache, 2.1e-4 with the custom-op cache and the ring), and a 2k 8da4w export (1,239,187,968 B, 597 s with the base model's solve on the same CPUs) answers the smoke test through 1.6.0's own runner. Nothing was published from it: the apps ship the 1.5.1 runtime, which `EXECUTORCH_VERSION` keeps every export on. LFM3-1.7B-Base (545fc8f4) was published on request (`--allow-base`; the chat-only default stands for the watcher): its fp32 model puts a newline first on all 12 calling rows (0.71 to 0.86) and is undecided on three of four quiet rows, so its gate tracks fp32 without measuring tool calling, and every window kept 12 of 12 (0.9786 against 0.76) and completed the smoke prefix with ' Paris'. Both repos' ten files match their reports' sha256 on the Hub. On a phone (Snapdragon 8 Elite Gen 5, SM8850, Android 16, through ExecuTorch's `llama_main` for arm64 built from main 98755608): the 2k file reasons and answers 'Paris' at 43 tok/s (1.87 GB resident), emits LFM3's XML `` for the app-shaped search prompt and answers the quiet one without a call; the 2k base file continues 'The capital of France is' with ' Paris.'; the 32k file reads a 5,870-token prompt in 2,047-token chunks (the ring wraps twice) and returns the code from its first line, prefill 307 tok/s, 2.62 GB resident. That last run needs a runner that chunks by the forward bound: the stock runner fails it, as it fails any prompt of 2,048 tokens or more on every `export_llm` file from 1.1.0 to 1.5.1 (the upstream bound bug, not LFM3's). [Evidence](evidence/lfm3-sm8850-llama_main.txt). | +| 43 | 2026-11-09 | XNNPACK | Qwen3-0.6B does not keep its tool decision at int4 under the gate that asks with reasoning switched off. A GPTQ solve (ExecuTorch 1.5.1) keeps 0 of 4 calling rows at every window, mean 0.016 against fp32's 0.619, and calls on 2 of 4 quiet rows; the October GPTQ file reads 0.086 and 0 of 4 on the same gate. The same model with fp32 linears (int8 embedding) reads 0.618 and 4 of 4. Qwen3-1.7B (12 of 12, 0.995 against 0.999) and Qwen3-4B (12 of 12, 0.9999) keep it at int4. | measured | verify logs of the rollout's twins; `pipeline audit Qwen/Qwen3-0.6B --context 2048` | Qwen3-0.6B ships fp32 linears on XNNPACK and no Vulkan folder. | +| 44 | 2026-11-09 | XNNPACK | `pipeline solve` on macOS arm64 (torch 2.14.0, Apple M5) turns a calibration row's hidden state into NaN after a few quantised layers: SmolLM2-135M at layer 4 (row 39), Qwen2.5-0.5B at layer 9, then Cholesky fails on the NaN Hessian. The NaN sits in the prompt positions the Hessian does not count, and no submodule output is non-finite. The same commit solves both on Linux x86 without it. The Mac's fp32 gate reference is off too: for SmolLM2-135M it grades 9 calling rows at 0.713 where Linux grades 4 at 0.583, and the published, gated GPTQ file scores 0.0002 against the Mac's reference and 0.585 against Linux's on the same Mac runner. | measured | local runs on the Mac against the rollout's EC2 solves | Solve and compute gate references on Linux only; a Mac can export and run a gate against a Linux reference. | +| 45 | 2026-11-09 | Both | XNNPACK and Vulkan `.pte` files hold their int4 weights in a form that can be read back and compared exactly: XNNPACK as `qbint4` values (unsigned nibbles around 8, low nibble first, bf16 scales) behind each fully-connected node's filter id; Vulkan as `linear_dq8ca_q4gsw` arguments (uint8 [out, in/2], fp32 scales [groups, out], int32 per-group code sums). Both serialise linears in execution order. Matched against fp32 sources, a published round-to-nearest file agrees with plain rounding on 99.0 to 99.2 percent of codes with every scale at amax/7.5; a GPTQ file on 67 to 87 percent, with 9 to 36 percent of scales at amax/7.5. | measured | SmolLM2-135M's published files; the rollout's byte checks | Byte checks before publish and after. | diff --git a/pipeline/__main__.py b/pipeline/__main__.py index 142ad72..86d7d7d 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -268,22 +268,51 @@ def _export_matrix(args) -> int: else: wanted = list(cfg.context_tiers) - entries = [] - for window in sorted({int(v) for v in wanted}, reverse=True): - need = sizing.host_need_bytes(arch, window, args.backend) - tier = export_xnnpack.pick_runner(arch, window, recipe.runner_tiers, args.backend) + def sized(window: int, backend: str, tiers) -> dict | None: + need = sizing.host_need_bytes(arch, window, backend) + tier = export_xnnpack.pick_runner(arch, window, tiers, backend) if tier is None: - print(f"no runner tier can build a {window}-token window (needs {need:,} B)", file=sys.stderr) - continue + print(f"{backend}: no runner tier can build a {window}-token window (needs {need:,} B)", file=sys.stderr) + return None entry = {"context": window, "runner": tier.label, "swap_gib": tier.swap_gib} if tier.label == settings.MODAL_TIER: # The Sandbox is sized to the window, not the tier: what the estimate needs plus # the margin, which pick_runner has already checked is within Modal's ceiling. entry["memory_mib"] = remote.memory_request(need, cfg.modal) if entry["memory_mib"] is None: - print(f"{window} needs more than Modal allows ({need:,} B)", file=sys.stderr) - continue - print(f"{window}: {tier.label}, needs {need / 2**30:.1f} GiB", file=sys.stderr) + print(f"{backend}: {window} needs more than Modal allows ({need:,} B)", file=sys.stderr) + return None + print(f"{backend} {window}: {tier.label}, needs {need / 2**30:.1f} GiB", file=sys.stderr) + return entry + + vulkan_ok = False + if args.with_vulkan: + # The XNNPACK window job also builds the Vulkan file from the same codes, gated + # through the XNNPACK file it has just checked (export-xnnpack.yml). Only for a model + # Vulkan can take, and in a Sandbox big enough for whichever of the two needs more. + if args.backend != "xnnpack": + print("--with-vulkan rides on the xnnpack matrix", file=sys.stderr) + return 2 + from pipeline import eligibility + + verdict = eligibility.evaluate(source, cfg) + vulkan_ok = not verdict.reasons and verdict.backends.get("vulkan") is None + if not vulkan_ok: + print(f"no Vulkan file: {verdict.backends.get('vulkan') or verdict.reasons}", file=sys.stderr) + + entries = [] + for window in sorted({int(v) for v in wanted}, reverse=True): + entry = sized(window, args.backend, recipe.runner_tiers) + if entry is None: + continue + if args.with_vulkan: + # The Vulkan file is built in a Sandbox of its own, after the XNNPACK one, so a GPU + # failure or timeout never costs the gated CPU file: it gets its own memory. A + # window Vulkan cannot be built at gets no GPU file rather than a squeezed one. + gpu = sized(window, "vulkan", cfg.vulkan.runner_tiers) if vulkan_ok else None + entry["vulkan"] = bool(gpu) and gpu.get("memory_mib") is not None + if entry["vulkan"]: + entry["vulkan_memory_mib"] = gpu["memory_mib"] entries.append(entry) print(json.dumps(entries)) return 0 @@ -356,7 +385,15 @@ def _export_qnn(args) -> int: def _publish_hf(args) -> int: from pipeline import publish - print(publish.publish_hf(Path(args.out), args.backend, args.target, args.context)) + print( + publish.publish_hf( + Path(args.out), + args.backend, + args.target, + args.context, + allow_unpublished_twin=getattr(args, "allow_unpublished_twin", False), + ) + ) return 0 @@ -374,7 +411,15 @@ def _correct_reports(args) -> int: def _publish_release(args) -> int: from pipeline import publish - print(publish.publish_release(Path(args.out), args.backend, args.target, args.context)) + print( + publish.publish_release( + Path(args.out), + args.backend, + args.target, + args.context, + allow_unpublished_twin=getattr(args, "allow_unpublished_twin", False), + ) + ) return 0 @@ -564,6 +609,11 @@ def main(argv: list[str] | None = None) -> int: export_matrix.add_argument("--revision", default="main") export_matrix.add_argument("--backend", default="vulkan", choices=["vulkan", "xnnpack"]) export_matrix.add_argument("--contexts", default="", help="JSON or comma-separated windows; empty for every tier") + export_matrix.add_argument( + "--with-vulkan", + action="store_true", + help="xnnpack only: mark each window whose job can also build the Vulkan file, and size it for both", + ) export_matrix.set_defaults(func=_export_matrix) solve_memory = commands.add_parser("solve-memory", help="MiB to request for the GPTQ solve on Modal") @@ -616,6 +666,12 @@ def main(argv: list[str] | None = None) -> int: command.add_argument("--backend", required=True, choices=["xnnpack", "vulkan", "qnn", "mtk"]) command.add_argument("--target", default=None) command.add_argument("--context", type=int, default=None, help="which window, when the folder holds several") + if name in ("publish-hf", "publish-release"): + command.add_argument( + "--allow-unpublished-twin", + action="store_true", + help="publish a GPTQ Vulkan file whose gate-proxy XNNPACK twin is deliberately not on the Hub", + ) command.set_defaults(func=func) args = parser.parse_args(argv) diff --git a/pipeline/publish.py b/pipeline/publish.py index 7301adb..4086a67 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -117,8 +117,77 @@ def require_publishable(report: dict) -> None: raise ValueError(f"{name} has GPTQ codes and no decision gate: verify it with --gate before publishing") +def require_twin_on_hub( + report: dict, by_path: dict[str, dict], existing: list[str], allow_unpublished_twin: bool = False +) -> None: + """A GPTQ Vulkan file publishes only into a head that holds the XNNPACK file its gate proxy + names: the same path, and a report of the same window giving the same sha256. Checked + against the very revision the commit is pinned to, on every retry, so another publisher + replacing the twin in between is caught (codex review, CI round 2). A Vulkan file gated + through a twin that is deliberately not published (SmolLM2-360M ships fp32 on XNNPACK) + needs ``allow_unpublished_twin``, said out loud.""" + proxy = report.get("gate_proxy") + if report.get("backend") != "vulkan" or not proxy or allow_unpublished_twin: + return + context = report["window"]["context"] + present = any( + path.startswith("xnnpack/") + and other["window"]["context"] == context + and other["files"][0]["sha256"] == proxy.get("sha256") + and other["files"][0]["path"] == proxy.get("file") + for path, other in by_path.items() + ) + if not present or proxy.get("file") not in existing: + raise ValueError( + f"{report['files'][0]['path']} is gated through {proxy.get('file')} ({str(proxy.get('sha256'))[:12]}), " + "which is not the XNNPACK file of this window on the Hub: publish that twin first, " + "or pass --allow-unpublished-twin if it is deliberately not published" + ) + + +def orphaned_gpu_window( + existing: list[str], deleted: set[str], by_path: dict[str, dict], report: dict, label: str +) -> list[str]: + """The Vulkan files of this window that the XNNPACK commit now being built leaves without + their gate, judged on the tree the commit produces (codex review, CI rounds 1 to 3): a + Vulkan report's gate proxy names its twin by path and sha256, and if that twin was on the + Hub and after this commit is gone, or there under other bytes, the Vulkan window goes with + it. A proxy naming a file that was never on the Hub (a twin deliberately not published, + as SmolLM2-360M's) is not this commit's to judge, and stays.""" + context = report["window"]["context"] + new_path, new_sha = report["files"][0]["path"], report["files"][0]["sha256"] + gpu_paths = [ + p + for p in (f"vulkan/{exporting.report_file(context)}", "vulkan/export-report.json") + if p in by_path and by_path[p]["window"]["context"] == context + ] + orphaned = [] + for gpu_path in gpu_paths: + proxy = by_path[gpu_path].get("gate_proxy") or {} + twin, twin_sha = proxy.get("file"), proxy.get("sha256") + if not twin or twin not in existing: + continue + if twin == new_path: + still_there = twin_sha == new_sha + else: + still_there = twin not in deleted + if not still_there: + orphaned.append(gpu_path) + if not orphaned: + return [] + files = [ + p for p in existing if p.startswith("vulkan/") and p.count("/") == 1 and window_of(p.split("/")[1]) == label + ] + return sorted(set(files) | set(orphaned)) + + def publish_hf( - out_dir: Path, backend: str, target: str | None = None, context: int | None = None, attempts: int = 6 + out_dir: Path, + backend: str, + target: str | None = None, + context: int | None = None, + attempts: int = 6, + allow_unpublished_twin: bool = False, ) -> str: """Commit this run's window into its backend folder beside the other windows, plus the shared root files; regenerate the folder's config.json and the repo README.md from @@ -160,11 +229,13 @@ def publish_hf( stale = [path for path in existing if superseded(path, folder, label)] reports = [report] migrated: dict[str, bytes] = {} + by_path: dict[str, dict] = {} for path in existing: if not is_report(path) or (path in stale and not path.endswith("/export-report.json")): continue local = hf_hub_download(repo_id, path, revision=info.sha, token=hf.token) other = json.loads(Path(local).read_text(encoding="utf-8")) + by_path[path] = other if path in stale: # A pre-matrix report carries no window in its name. Its window's file stays # unless it is the one this run replaces, so the report moves to the labelled name. @@ -174,6 +245,20 @@ def publish_hf( json.dumps(other, indent=2).encode() + b"\n" ) reports.append(other) + if folder == "vulkan": + require_twin_on_hub(report, by_path, existing, allow_unpublished_twin) + # Replacing an XNNPACK window takes with it the Vulkan window whose gate proxy names the + # file being replaced: left up, that GPU file would cite a gate no longer on the Hub (a + # failed or skipped Vulkan build, an fp32 or rounded rerun). Its new twin, when one is + # built, publishes right after this commit (export-xnnpack.yml). + orphaned = ( + orphaned_gpu_window(existing, set(stale) - set(new_paths), by_path, report, label) + if folder == "xnnpack" + else [] + ) + if orphaned: + dropped = {id(by_path[p]) for p in orphaned if p in by_path} + reports = [r for r in reports if id(r) not in dropped] folder_reports = [r for r in reports if _folder(r["backend"], r.get("target")) == folder] config = json.dumps(manifest.backend_config(folder_reports), indent=2) + "\n" license_files = sorted({f for r in reports for f in r["source"].get("license_files", [])}) @@ -189,11 +274,23 @@ def publish_hf( operations += [CommitOperationAdd(path_in_repo=k, path_or_fileobj=v) for k, v in migrated.items()] operations += [CommitOperationAdd(path_in_repo=k, path_or_fileobj=str(v)) for k, v in root_files.items()] operations.append(CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme.encode("utf-8"))) + message = f"{folder} {label}: {report['source']['id']}@{report['source']['sha'][:12]}" + if orphaned: + operations += [CommitOperationDelete(path_in_repo=p) for p in orphaned] + gpu_reports = [r for r in reports if _folder(r["backend"], r.get("target")) == "vulkan"] + if gpu_reports: + gpu_config = json.dumps(manifest.backend_config(gpu_reports), indent=2) + "\n" + operations.append( + CommitOperationAdd(path_in_repo="vulkan/config.json", path_or_fileobj=gpu_config.encode()) + ) + elif "vulkan/config.json" in existing: + operations.append(CommitOperationDelete(path_in_repo="vulkan/config.json")) + message += f"; withdraws vulkan {label}, whose gate was the file replaced" try: commit = hf.create_commit( repo_id, operations, - commit_message=f"{folder} {label}: {report['source']['id']}@{report['source']['sha'][:12]}", + commit_message=message, parent_commit=info.sha, ) return commit.commit_url @@ -238,13 +335,66 @@ def release_notes(report: dict, hf_url: str | None) -> str: return "\n".join(lines) + "\n" +def require_twin_on_hub_now(report: dict) -> None: + """``require_twin_on_hub`` against the Hub's current head: a release is a mirror of the + Hub, so a GPU release goes out only while the twin its gate cites is there (codex review, + CI round 3).""" + from huggingface_hub import hf_hub_download + + hf = hub.api() + info = hf.model_info(report["output_repo"], expand=["sha", "siblings"]) + existing = [s.rfilename for s in info.siblings or []] + by_path = {} + for path in existing: + if path.startswith("xnnpack/") and is_report(path): + local = hf_hub_download(report["output_repo"], path, revision=info.sha, token=hf.token) + by_path[path] = json.loads(Path(local).read_text(encoding="utf-8")) + require_twin_on_hub(report, by_path, existing) + + +def withdraw_orphaned_gpu_release(report: dict) -> str | None: + """After an XNNPACK release is (re)published: the Vulkan release of the same window and + source revision, if its gate proxy names this very file under other bytes, cites a gate + this release no longer holds, and is deleted (codex review, CI round 3). Returns its tag.""" + import tempfile + + tag = release_tag({**report, "backend": "vulkan", "target": None}) + view = subprocess.run(["gh", "release", "view", tag, "--json", "tagName"], capture_output=True, text=True) + if view.returncode != 0: + # Only a release that is confirmed missing means there is nothing to withdraw; an API, + # network or auth failure must not pass for one (codex review, CI round 4). + if "release not found" in (view.stderr + view.stdout).lower(): + return None + raise RuntimeError(f"gh release view {tag} failed: {view.stderr.strip()[-500:]}") + with tempfile.TemporaryDirectory() as tmp: + subprocess.run( + ["gh", "release", "download", tag, "--pattern", "export-report-*.json", "--dir", tmp], + capture_output=True, + check=True, + ) + found = sorted(Path(tmp).glob("export-report-*.json")) + proxy = (json.loads(found[0].read_text(encoding="utf-8")).get("gate_proxy") or {}) if found else {} + twin = report["files"][0] + if proxy.get("file") != twin["path"] or proxy.get("sha256") == twin["sha256"]: + return None + subprocess.run(["gh", "release", "delete", tag, "--yes", "--cleanup-tag"], check=True) + return tag + + def publish_release( - out_dir: Path, backend: str, target: str | None = None, context: int | None = None, attempts: int = 5 + out_dir: Path, + backend: str, + target: str | None = None, + context: int | None = None, + attempts: int = 5, + allow_unpublished_twin: bool = False, ) -> str: """Create (or update) a GitHub release with this window's files under the 2 GiB limit.""" report = load_report(out_dir, backend, target, context) require_publishable(report) require_same_bytes(out_dir, report) + if backend == "vulkan" and report.get("gate_proxy") and not allow_unpublished_twin: + require_twin_on_hub_now(report) folder = out_dir / _folder(backend, target) tag = release_tag(report) label = naming.window_label(report["window"]["context"]) @@ -260,6 +410,8 @@ def publish_release( for attempt in range(attempts): created = subprocess.run(create, capture_output=True, text=True) if created.returncode == 0: + if backend == "xnnpack": + withdraw_orphaned_gpu_release(report) return tag output = created.stderr + created.stdout if "already exists" in output: @@ -268,9 +420,15 @@ def publish_release( raise RuntimeError(f"gh release create failed: {created.stderr.strip()[-2000:]}") time.sleep(30 * (attempt + 1)) # The release exists (a re-run of the same window, or a create that answered 5xx after - # creating it): replace its assets and notes. + # creating it): replace its assets and notes, and with them the GPU release they gated. subprocess.run(["gh", "release", "upload", tag, "--clobber", *assets], check=True) - subprocess.run(["gh", "release", "edit", tag, "--notes", notes], check=True) + try: + subprocess.run(["gh", "release", "edit", tag, "--notes", notes], check=True) + finally: + # The assets are replaced already: the GPU release they gated goes even if the notes + # edit failed (codex review, CI round 4). + if backend == "xnnpack": + withdraw_orphaned_gpu_release(report) return tag diff --git a/pipeline/watch.py b/pipeline/watch.py index f5d5c0e..d158c17 100644 --- a/pipeline/watch.py +++ b/pipeline/watch.py @@ -33,12 +33,17 @@ # joins in phase 4. WORKFLOWS = { "xnnpack": "export-xnnpack.yml", - "vulkan": "export-vulkan.yml", "qnn": "export-qnn.yml", "mtk": "export-mtk.yml", } -# Dispatch order: the CPU exports of every model first, then GPU, then each NPU in turn. -STAGES = ("xnnpack", "vulkan", "qnn", "mtk") +# Backends with no workflow of their own, built in another backend's run. Vulkan files come +# out of the XNNPACK workflow's window jobs (2026-11-09): the same solved codes, gated through +# the XNNPACK file of the same window, since the host has no Vulkan runtime to gate them. A +# workflow of its own could only solve again or round to nearest, which every published Vulkan +# file was until then. +BUILT_WITH = {"vulkan": "xnnpack"} +# Dispatch order: the CPU (and with it GPU) exports of every model first, then each NPU in turn. +STAGES = ("xnnpack", "qnn", "mtk") # A model whose metadata could not be read is retried this many runs before it is skipped. MAX_ATTEMPTS = 5 @@ -169,10 +174,17 @@ def _evaluate( entry["reasons"] = verdict.reasons entry["backends"] = {} for backend, why in verdict.backends.items(): - if not verdict.reasons and why is None and backend in WORKFLOWS: + if not verdict.reasons and why is None and (backend in WORKFLOWS or backend in BUILT_WITH): entry["backends"][backend] = {"status": "pending"} else: entry["backends"][backend] = {"status": "skipped", "reason": why or "no export workflow yet"} + for backend, host in BUILT_WITH.items(): + info, host_info = entry["backends"].get(backend), entry["backends"].get(host) + if info and info["status"] == "pending" and (host_info or {}).get("status") != "pending": + entry["backends"][backend] = { + "status": "skipped", + "reason": f"built in the {host} run, which this model does not get", + } pending = any(b["status"] == "pending" for b in entry["backends"].values()) entry["status"] = "pending" if pending else "skipped" entry.pop("attempts", None) @@ -266,9 +278,36 @@ def run( continue info.update(status="dispatched", at=stamp) info.pop("last_error", None) + # The backends this run builds too (Vulkan with XNNPACK) go out with it. An entry + # recorded before the backend existed has no line for it at all (26 such models on + # the state branch, 2026-11-09): the run's targets job decides whether the model + # can take it, and the state says it went out with the run. + for built, host in BUILT_WITH.items(): + if host != backend: + continue + other = entry["backends"].get(built) + if other is None: + entry["backends"][built] = { + "status": "dispatched", + "at": stamp, + "built_with": backend, + "note": "entry predates this backend; the run decides whether the model can take it", + } + elif other["status"] == "pending": + other.update(status="dispatched", at=stamp, built_with=backend) summary["dispatched"].append(f"{model_id} -> {backend}") budget -= 1 break # later stages wait for this one + # A model whose host run went out before its backend was built there (a Vulkan stage + # recorded under the old staging) would wait forever: say what would build it instead. + for entry in models.values(): + for built, host in BUILT_WITH.items(): + info, host_info = entry.get("backends", {}).get(built), entry.get("backends", {}).get(host) + if info and info["status"] == "pending" and (host_info or {}).get("status") not in (None, "pending"): + info.update( + status="skipped", + reason=f"its {host} run predates {built} being built there; backfill the model to build it", + ) for entry in models.values(): if entry.get("status") == "pending" and all( i["status"] != "pending" for i in entry.get("backends", {}).values() diff --git a/tests/test_export_xnnpack.py b/tests/test_export_xnnpack.py index ce8cd19..94a6037 100644 --- a/tests/test_export_xnnpack.py +++ b/tests/test_export_xnnpack.py @@ -1,4 +1,5 @@ import dataclasses +import json import pytest from conftest import make_source @@ -189,3 +190,47 @@ def test_an_int8_head_is_priced_at_a_byte_a_weight(source): assert 0.4 * head < eight - four < 0.5 * head # (1 - 0.5625) bytes per weight, plus overhead rows = sizing.choose_context(arch, (2048,), 5_000_000_000, 500_000_000, None, output_bits=8).table assert rows[0]["device_resident_bytes"] == sizing.device_resident_bytes(arch, 2048, 500_000_000, output_bits=8) + + +def matrix(monkeypatch, capsys, model_id, config, *flags): + from pipeline import hub + from pipeline.__main__ import main + + monkeypatch.setattr(hub, "fetch", lambda m, r: make_source(model_id, config=config)) + rc = main(["export-matrix", model_id, "--backend", "xnnpack", *flags]) + return rc, json.loads(capsys.readouterr().out.strip().splitlines()[-1]) if rc == 0 else None + + +def test_the_xnnpack_matrix_marks_the_windows_whose_job_also_builds_the_vulkan_file(monkeypatch, capsys): + from conftest import hf_config + + config = hf_config("Qwen/Qwen3-1.7B") + rc, plain = matrix(monkeypatch, capsys, "Qwen/Qwen3-1.7B", config) + rc, both = matrix(monkeypatch, capsys, "Qwen/Qwen3-1.7B", config, "--with-vulkan") + assert rc == 0 and [e["context"] for e in both] == [e["context"] for e in plain] + assert all(e["vulkan"] for e in both) + # The Vulkan file gets a Sandbox of its own: the XNNPACK one keeps its size. + for p, b in zip(plain, both, strict=True): + assert b["memory_mib"] == p["memory_mib"] and b["vulkan_memory_mib"] > 0 + assert "vulkan" not in plain[0] + + +def test_a_model_vulkan_cannot_take_gets_no_vulkan_file(monkeypatch, capsys): + from conftest import load_json + + config = load_json("lfm2.5-350m.config.json") # LFM2.5: no Vulkan path (its hybrid segfaults there) + rc, plain = matrix(monkeypatch, capsys, "LiquidAI/LFM2.5-350M", config) + rc, both = matrix(monkeypatch, capsys, "LiquidAI/LFM2.5-350M", config, "--with-vulkan") + assert rc == 0 and both and not any(e["vulkan"] for e in both) + assert not any("vulkan_memory_mib" in e for e in both) + assert [e["memory_mib"] for e in both] == [e["memory_mib"] for e in plain] + + +def test_with_vulkan_belongs_to_the_xnnpack_matrix(monkeypatch, capsys): + from conftest import hf_config + + from pipeline import hub + from pipeline.__main__ import main + + monkeypatch.setattr(hub, "fetch", lambda m, r: make_source("Qwen/Qwen3-1.7B", config=hf_config("Qwen/Qwen3-1.7B"))) + assert main(["export-matrix", "Qwen/Qwen3-1.7B", "--backend", "vulkan", "--with-vulkan"]) == 2 diff --git a/tests/test_publish.py b/tests/test_publish.py index 269bf12..594f0c3 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -2,6 +2,7 @@ import hashlib import json +from pathlib import Path from types import SimpleNamespace import pytest @@ -330,3 +331,219 @@ def test_a_release_never_attaches_a_model_file_its_report_does_not_list(tmp_path monkeypatch.setattr(publish.subprocess, "run", lambda *a, **k: pytest.fail("reached gh")) with pytest.raises(ValueError, match="renamed.pte.*not in the report"): publish.publish_release(out, "xnnpack") + + +def publish_over(tmp_path, monkeypatch, remote, new_sha=None): + """Publish a new xnnpack 4k window over ``remote``; return (deleted, added, message).""" + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + new = report(4096) + if new_sha: + new["files"][0]["sha256"] = new_sha + (out / "xnnpack" / "Qwen3-1.7B-8da4w-4k.pte").write_bytes(b"pte") + (out / "xnnpack" / "export-report-4k.json").write_text(json.dumps(new)) + (out / "xnnpack" / "config.json").write_text(json.dumps(manifest.backend_config(new))) + hub = FakeHub(remote) + monkeypatch.setattr(publish.hub, "api", lambda: hub) + + def fake_download(repo_id, path, revision, token): + local = tmp_path / "dl" / path + local.parent.mkdir(parents=True, exist_ok=True) + local.write_text(json.dumps(remote[path])) + return str(local) + + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download) + monkeypatch.setattr(publish, "require_same_bytes", lambda out_dir, report: None) + publish.publish_hf(out, "xnnpack") + [(operations, message, _)] = hub.commits + deleted = sorted(op.path_in_repo for op in operations if type(op).__name__ == "CommitOperationDelete") + added = {op.path_in_repo: op for op in operations if type(op).__name__ == "CommitOperationAdd"} + return deleted, added, message + + +def gpu(context, twin_sha): + r = report(context, backend="vulkan") + r["gate_proxy"] = {"file": f"xnnpack/Qwen3-1.7B-8da4w-{context // 1024}k.pte", "sha256": twin_sha} + return r + + +def remote_with_gpu(proxy_4k): + old = report(4096) + old["files"][0]["sha256"] = "old-xnnpack-4k" + return { + "README.md": None, + "xnnpack/Qwen3-1.7B-8da4w-4k.pte": None, + "xnnpack/export-report-4k.json": old, + "xnnpack/config.json": None, + "vulkan/Qwen3-1.7B-8da4w-4k.pte": None, + "vulkan/export-report-4k.json": gpu(4096, proxy_4k), + "vulkan/Qwen3-1.7B-8da4w-8k.pte": None, + "vulkan/export-report-8k.json": gpu(8192, "xnnpack-8k"), + "vulkan/config.json": None, + } + + +def test_replacing_an_xnnpack_window_withdraws_the_vulkan_window_gated_through_it(tmp_path, monkeypatch): + deleted, added, message = publish_over(tmp_path, monkeypatch, remote_with_gpu("old-xnnpack-4k")) + assert deleted == ["vulkan/Qwen3-1.7B-8da4w-4k.pte", "vulkan/export-report-4k.json"] + assert "withdraws vulkan 4k" in message + # The other Vulkan window stays, and the folder's config and the README forget the 4k one. + config = json.loads(added["vulkan/config.json"].path_or_fileobj) + assert [v["context"] for v in config["variants"]] == [8192] + readme = added["README.md"].path_or_fileobj.decode() + assert "vulkan/Qwen3-1.7B-8da4w-8k.pte" in readme and "vulkan/Qwen3-1.7B-8da4w-4k.pte" not in readme + + +@pytest.mark.parametrize("case", ["unpublished-twin", "same-file"]) +def test_a_vulkan_window_whose_twin_survives_the_commit_stays(tmp_path, monkeypatch, case): + if case == "unpublished-twin": + # SmolLM2-360M: gated through a GPTQ build that was never on the Hub (its CPU files are fp32). + remote = remote_with_gpu("never-published") + remote["vulkan/export-report-4k.json"]["gate_proxy"]["file"] = "xnnpack/Qwen3-1.7B-8da4w-gptq-4k.pte" + new_sha = None + else: + # The very file being republished: same path, same bytes. + remote, new_sha = remote_with_gpu("same-bytes"), "same-bytes" + deleted, added, message = publish_over(tmp_path, monkeypatch, remote, new_sha=new_sha) + assert not any(p.startswith("vulkan/") for p in deleted) and "withdraws" not in message + assert "vulkan/config.json" not in added + + +def publish_gpu(tmp_path, monkeypatch, remote, **kwargs): + """Publish a GPTQ Vulkan 4k window gated through xnnpack 4k sha 'twin-4k' over ``remote``.""" + out = tmp_path / "gpu-out" + (out / "vulkan").mkdir(parents=True, exist_ok=True) + new = gpu(4096, "twin-4k") + (out / "vulkan" / "Qwen3-1.7B-8da4w-4k.pte").write_bytes(b"pte") + (out / "vulkan" / "export-report-4k.json").write_text(json.dumps(new)) + (out / "vulkan" / "config.json").write_text(json.dumps(manifest.backend_config(new))) + hub = FakeHub(remote) + monkeypatch.setattr(publish.hub, "api", lambda: hub) + + def fake_download(repo_id, path, revision, token): + local = tmp_path / "dl" / path + local.parent.mkdir(parents=True, exist_ok=True) + local.write_text(json.dumps(remote[path])) + return str(local) + + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download) + return publish.publish_hf(out, "vulkan", **kwargs), hub + + +def remote_twin(sha): + twin = report(4096) + twin["files"][0]["sha256"] = sha + return {"xnnpack/Qwen3-1.7B-8da4w-4k.pte": None, "xnnpack/export-report-4k.json": twin, "README.md": None} + + +def test_a_vulkan_file_publishes_only_beside_the_twin_its_proxy_names(tmp_path, monkeypatch): + url, hub = publish_gpu(tmp_path, monkeypatch, remote_twin("twin-4k")) + assert url == "https://hf/commit/1" and len(hub.commits) == 1 + + +def test_a_vulkan_file_whose_twin_was_replaced_meanwhile_is_refused(tmp_path, monkeypatch): + with pytest.raises(ValueError, match="not the XNNPACK file of this window on the Hub"): + publish_gpu(tmp_path, monkeypatch, remote_twin("someone-else-replaced-it")) + with pytest.raises(ValueError, match="publish that twin first"): + publish_gpu(tmp_path, monkeypatch, {"README.md": None}) + # A twin that is deliberately not published (SmolLM2-360M) is said out loud. + url, _ = publish_gpu(tmp_path, monkeypatch, {"README.md": None}, allow_unpublished_twin=True) + assert url == "https://hf/commit/1" + + +def test_a_pre_matrix_xnnpack_report_still_withdraws_the_vulkan_window_it_gated(tmp_path, monkeypatch): + remote = remote_with_gpu("old-xnnpack-4k") + old = remote.pop("xnnpack/export-report-4k.json") + remote["xnnpack/export-report.json"] = old # the unlabelled report of an older layout + deleted, _, message = publish_over(tmp_path, monkeypatch, remote) + assert "vulkan/Qwen3-1.7B-8da4w-4k.pte" in deleted and "withdraws vulkan 4k" in message + + +def test_a_replaced_twin_takes_its_vulkan_window_with_it_even_without_its_report(tmp_path, monkeypatch): + # Judged on the tree the commit produces, not on old reports: the proxy's file is replaced. + remote = remote_with_gpu("old-xnnpack-4k") + del remote["xnnpack/export-report-4k.json"] + deleted, _, message = publish_over(tmp_path, monkeypatch, remote) + assert "vulkan/Qwen3-1.7B-8da4w-4k.pte" in deleted and "withdraws vulkan 4k" in message + + +def test_the_same_bytes_under_another_name_still_withdraw_the_vulkan_window(tmp_path, monkeypatch): + # The proxy names a path this commit deletes; matching the sha alone would have kept it. + remote = remote_with_gpu(hashlib.sha256(b"pte").hexdigest()) + remote["vulkan/export-report-4k.json"]["gate_proxy"]["file"] = "xnnpack/Qwen3-1.7B-8da4w-old-name-4k.pte" + remote["xnnpack/Qwen3-1.7B-8da4w-old-name-4k.pte"] = None + deleted, _, message = publish_over(tmp_path, monkeypatch, remote) + assert "xnnpack/Qwen3-1.7B-8da4w-old-name-4k.pte" in deleted and "vulkan/export-report-4k.json" in deleted + + +class FakeGh: + """`gh release view/download/delete` for one Vulkan release holding one report.""" + + def __init__(self, gpu_report=None): + self.gpu_report = gpu_report + self.calls = [] + + def run(self, command, **kwargs): + self.calls.append(command[:3]) + if command[:3] == ["gh", "release", "view"]: + if self.gpu_report == "api-error": + return SimpleNamespace(returncode=1, stdout="", stderr="HTTP 502: Bad Gateway") + if self.gpu_report is None: + return SimpleNamespace(returncode=1, stdout="", stderr="release not found") + return SimpleNamespace(returncode=0, stdout="", stderr="") + if command[:3] == ["gh", "release", "download"]: + directory = command[command.index("--dir") + 1] + (Path(directory) / "export-report-4k.json").write_text(json.dumps(self.gpu_report)) + return SimpleNamespace(returncode=0, stdout="", stderr="") + if command[:3] == ["gh", "release", "delete"]: + return SimpleNamespace(returncode=0, stdout="", stderr="") + raise AssertionError(command) + + +@pytest.mark.parametrize( + ("gpu_report", "withdrawn"), + [ + (None, False), # no GPU release for this window + (gpu(4096, hashlib.sha256(b"pte").hexdigest()), False), # gated through these very bytes + (gpu(4096, "older-bytes"), True), # gated through this file before it was re-released + ({**gpu(4096, "older-bytes"), "gate_proxy": {"file": "xnnpack/other.pte", "sha256": "x"}}, False), + ], +) +def test_re_releasing_an_xnnpack_window_withdraws_only_the_gpu_release_it_gated(monkeypatch, gpu_report, withdrawn): + gh = FakeGh(gpu_report) + monkeypatch.setattr(publish.subprocess, "run", gh.run) + tag = publish.withdraw_orphaned_gpu_release(report(4096)) + assert (tag == "Qwen3-1.7B-vulkan-4k-abcdef0") is withdrawn + assert (["gh", "release", "delete"] in gh.calls) is withdrawn + + +def test_a_gpu_release_needs_its_twin_on_the_hub(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "vulkan").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "vulkan" / "Qwen3-1.7B-8da4w-4k.pte").write_bytes(b"pte") + (out / "vulkan" / "export-report-4k.json").write_text(json.dumps(gpu(4096, "twin-4k"))) + checked = [] + + def missing_twin(report): + checked.append(report["files"][0]["path"]) + raise ValueError("not the XNNPACK file of this window on the Hub") + + monkeypatch.setattr(publish, "require_twin_on_hub_now", missing_twin) + with pytest.raises(ValueError, match="not the XNNPACK file"): + publish.publish_release(out, "vulkan") + assert checked == ["vulkan/Qwen3-1.7B-8da4w-4k.pte"] + # Said out loud, it goes past the check (to gh, which this test stops at). + monkeypatch.setattr(publish.subprocess, "run", lambda *a, **k: SimpleNamespace(returncode=0, stdout="", stderr="")) + assert publish.publish_release(out, "vulkan", allow_unpublished_twin=True) == "Qwen3-1.7B-vulkan-4k-abcdef0" + + +def test_an_unreadable_gpu_release_is_an_error_not_an_absent_one(monkeypatch): + monkeypatch.setattr(publish.subprocess, "run", FakeGh("api-error").run) + with pytest.raises(RuntimeError, match="gh release view"): + publish.withdraw_orphaned_gpu_release(report(4096)) diff --git a/tests/test_remote.py b/tests/test_remote.py index 2836a2f..f415506 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -70,7 +70,7 @@ def test_the_image_carries_what_the_pipeline_reads(): assert {"pipeline", "config", "third_party", "requirements"} <= set(remote.REPO_DIRS) -@pytest.mark.parametrize("workflow", ["export-xnnpack.yml", "export-vulkan.yml"]) +@pytest.mark.parametrize("workflow", ["export-xnnpack.yml"]) def test_the_workflow_sends_each_window_to_modal_with_its_own_memory(workflow): text = (settings.ROOT / ".github/workflows" / workflow).read_text(encoding="utf-8") assert "python -m pipeline modal-run --memory-mib" in text @@ -184,3 +184,35 @@ def test_the_container_peak_is_read_from_its_cgroup(tmp_path): sampler.sample() assert sampler.result()["peak_cgroup_bytes"] == 3221225472 assert MemorySampler(meminfo=tmp_path / "missing", cgroup=()).result()["peak_cgroup_bytes"] is None + + +def test_vulkan_files_are_built_only_from_solved_codes_in_the_xnnpack_window_jobs(): + workflows = settings.ROOT / ".github/workflows" + # No workflow of its own: one could only solve again or round to nearest. + assert not (workflows / "export-vulkan.yml").exists() + text = (workflows / "export-xnnpack.yml").read_text(encoding="utf-8") + # Asked for only on a run that solves 8da4w codes, never a rounded or fp32 one. + lines = text.splitlines() + guard = lines[next(i for i, line in enumerate(lines) if 'WITH_VULKAN="--with-vulkan"' in line) - 1] + assert '"$VULKAN" = "true"' in guard and '"$GPTQ" = "true"' in guard and '"$QMODE" = "8da4w"' in guard + assert '--backend xnnpack --contexts "$CONTEXTS" $WITH_VULKAN' in text + # Every Vulkan export in every workflow carries the solve's codes and its XNNPACK twin. + for wf in workflows.glob("*.yml"): + body = wf.read_text(encoding="utf-8") + for chunk in body.split("python -m pipeline export-vulkan")[1:]: + command = " ".join(chunk.split("\n")[:2]) + assert '--codes "$IN/codes.pt"' in command and "--gate-proxy" in command, wf.name + # The CPU file is published before the GPU Sandbox starts, in a Sandbox of its own, so a + # GPU failure or timeout never costs it; the GPU file publishes only once its twin is out. + publish_cpu = text.index('publish-hf "$WORK_DIR/out" --backend xnnpack') + gpu_sandbox = text.index('--memory-mib "$VULKAN_MEMORY_MIB"') + publish_gpu = text.index('publish-hf "$WORK_DIR/vulkan-out" --backend vulkan') + assert publish_cpu < gpu_sandbox < publish_gpu + gpu_step = text[text.rindex("- name:", 0, publish_gpu) : publish_gpu] + assert "steps.publish_xnnpack.outcome == 'success'" in gpu_step and "env.VULKAN_BUILT == '1'" in gpu_step + assert "VULKAN_FAILED=1" in text and "env.VULKAN_FAILED == '1'" in text + # Its GitHub release, likewise, only beside its twin's and after its own Hub publish. + release_gpu = text.index('publish-release "$WORK_DIR/vulkan-out" --backend vulkan') + release_step = text[text.rindex("- name:", 0, release_gpu) : release_gpu] + assert "steps.release_xnnpack.outcome == 'success'" in release_step + assert "(!inputs.publish_hf || steps.publish_vulkan.outcome == 'success')" in release_step diff --git a/tests/test_runner_tiers.py b/tests/test_runner_tiers.py index 3023ae8..602bfdb 100644 --- a/tests/test_runner_tiers.py +++ b/tests/test_runner_tiers.py @@ -1,4 +1,4 @@ -"""Per-window runner choice for the export_llm backends (export-vulkan.yml's matrix), on a +"""Per-window runner choice for the export_llm backends (export-xnnpack.yml's matrix), on a three-size fleet; what the configured tier itself carries is test_namespace_runner.py.""" import pytest diff --git a/tests/test_watch.py b/tests/test_watch.py index 91ca65e..ae22241 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -81,16 +81,17 @@ def test_new_models_are_checked_once_and_dispatched(tmp_path): entry = models["Qwen/Qwen3-1.7B-Instruct-2609"] assert entry["status"] == "pending" # its MediaTek stage is still to come assert entry["backends"]["xnnpack"]["status"] == "dispatched" + # Vulkan is built in the XNNPACK run, from the same codes: it goes out with it. + assert entry["backends"]["vulkan"]["status"] == "dispatched" + assert entry["backends"]["vulkan"]["built_with"] == "xnnpack" assert entry["backends"]["qnn"]["status"] == "skipped" assert entry["backends"]["mtk"]["status"] == "pending" # Seeded models are not re-checked. assert "Qwen/Qwen3-0.6B" not in summary["new"] - # The next passes find nothing new; each idle stage hands over: Vulkan, then MediaTek. + # The next pass finds nothing new; the idle XNNPACK stage hands over to MediaTek. hub.dispatched.clear() summary = run(tmp_path, hub) - assert summary["new"] == [] and [d[0] for d in hub.dispatched] == ["export-vulkan.yml"] - summary = run(tmp_path, hub) - assert [d[0] for d in hub.dispatched] == ["export-vulkan.yml", "export-mtk.yml"] + assert summary["new"] == [] and [d[0] for d in hub.dispatched] == ["export-mtk.yml"] assert summary["state"]["models"]["Qwen/Qwen3-1.7B-Instruct-2609"]["status"] == "dispatched" @@ -137,13 +138,10 @@ def test_backfill_exports_a_seeded_model(tmp_path): summary = run(tmp_path, hub, backfill=("Qwen/Qwen3-1.7B",)) assert summary["backfilled"] == ["Qwen/Qwen3-1.7B"] # Qwen3-1.7B is also in ExecuTorch's Qualcomm registry, but stages go one backend at a - # time: XNNPACK now, Vulkan and QNN on the passes after the previous stage is idle. + # time: XNNPACK (and Vulkan with it) now, QNN on the pass after that stage is idle. assert hub.dispatched == [("export-xnnpack.yml", "Qwen/Qwen3-1.7B", "sha-Qwen/Qwen3-1.7B")] assert summary["stage"] == "xnnpack" summary = run(tmp_path, hub) - assert hub.dispatched[-1] == ("export-vulkan.yml", "Qwen/Qwen3-1.7B", "sha-Qwen/Qwen3-1.7B") - assert summary["stage"] == "vulkan" - summary = run(tmp_path, hub) assert hub.dispatched[-1] == ("export-qnn.yml", "Qwen/Qwen3-1.7B", "sha-Qwen/Qwen3-1.7B") assert summary["stage"] == "qnn" @@ -217,11 +215,7 @@ def test_stages_wait_for_the_previous_backend_to_finish(tmp_path): hub.dispatched.clear() summary = run(tmp_path, hub, in_flight=lambda wf: wf == "export-xnnpack.yml") assert hub.dispatched == [] and summary["stage"] == "xnnpack" - # XNNPACK idle: Vulkan for every model, then (next pass) QNN for the two in the registry. - summary = run(tmp_path, hub) - assert {d[0] for d in hub.dispatched} == {"export-vulkan.yml"} and len(hub.dispatched) == 3 - assert summary["stage"] == "vulkan" - hub.dispatched.clear() + # XNNPACK idle (Vulkan was built in those runs): QNN for the two in the registry. summary = run(tmp_path, hub) assert sorted(d[1] for d in hub.dispatched) == ["Qwen/Qwen3-0.6B", "Qwen/Qwen3-1.7B"] assert {d[0] for d in hub.dispatched} == {"export-qnn.yml"} and summary["stage"] == "qnn" @@ -268,3 +262,65 @@ def test_state_version_is_checked(tmp_path): (tmp_path / "seen.json").write_text(json.dumps({"version": 99, "models": {}})) with pytest.raises(ValueError, match="state version"): watch.load_state(tmp_path / "seen.json") + + +def test_vulkan_is_never_dispatched_on_its_own_and_comes_back_with_a_requeue(tmp_path): + # Vulkan files are built in the XNNPACK run from the same solved codes; a workflow of its + # own could only solve again or round to nearest. + assert "vulkan" not in watch.WORKFLOWS and "vulkan" not in watch.STAGES + hub = FakeHub({"Qwen": []}) + run(tmp_path, hub) + hub.listings["Qwen"] = ["Qwen/Qwen3-1.7B"] + run(tmp_path, hub) + assert hub.dispatched == [("export-xnnpack.yml", "Qwen/Qwen3-1.7B", "sha-Qwen/Qwen3-1.7B")] + # Cancelled runs go back to the queue together, and out again together. + hub.dispatched.clear() + summary = run(tmp_path, hub, requeue=True) + assert hub.dispatched == [("export-xnnpack.yml", "Qwen/Qwen3-1.7B", "sha-Qwen/Qwen3-1.7B")] + vulkan = summary["state"]["models"]["Qwen/Qwen3-1.7B"]["backends"]["vulkan"] + assert vulkan["status"] == "dispatched" and vulkan["built_with"] == "xnnpack" + + +def test_a_vulkan_stage_left_pending_by_the_old_staging_is_skipped_with_the_way_to_build_it(tmp_path): + hub = FakeHub({"Qwen": []}) + run(tmp_path, hub) + hub.listings["Qwen"] = ["Qwen/Qwen3-1.7B"] + run(tmp_path, hub) + # A state written before 2026-11-09: XNNPACK out, Vulkan waiting for a stage that is gone. + state = json.loads((tmp_path / "seen.json").read_text()) + state["models"]["Qwen/Qwen3-1.7B"]["backends"]["vulkan"] = {"status": "pending"} + (tmp_path / "seen.json").write_text(json.dumps(state)) + hub.dispatched.clear() + summary = run(tmp_path, hub) + vulkan = summary["state"]["models"]["Qwen/Qwen3-1.7B"]["backends"]["vulkan"] + assert vulkan["status"] == "skipped" and "backfill" in vulkan["reason"] + assert all(d[0] != "export-vulkan.yml" for d in hub.dispatched) + + +def test_vulkan_is_skipped_when_the_xnnpack_run_that_builds_it_is_not_made(monkeypatch): + class Verdict: + reasons, family = [], "qwen3" + backends = {"xnnpack": "not here", "vulkan": None, "qnn": None, "mtk": None} + + monkeypatch.setattr(watch.eligibility, "evaluate", lambda *a, **k: Verdict()) + entry = {"first_seen": "x"} + source = make_source("Qwen/Qwen3-1.7B", config=QWEN3) + watch._evaluate("Qwen/Qwen3-1.7B", "main", CFG, lambda m, r: source, entry) + assert entry["backends"]["vulkan"]["status"] == "skipped" + assert "xnnpack run" in entry["backends"]["vulkan"]["reason"] + + +def test_an_entry_from_before_vulkan_existed_records_it_when_its_xnnpack_run_goes_out_again(tmp_path): + # The real shape on the state branch: a dispatched model whose backends have no vulkan line. + hub = FakeHub({"Qwen": []}) + run(tmp_path, hub) + hub.listings["Qwen"] = ["Qwen/Qwen3-1.7B"] + run(tmp_path, hub) + state = json.loads((tmp_path / "seen.json").read_text()) + del state["models"]["Qwen/Qwen3-1.7B"]["backends"]["vulkan"] + (tmp_path / "seen.json").write_text(json.dumps(state)) + hub.dispatched.clear() + summary = run(tmp_path, hub, requeue=True) + assert hub.dispatched == [("export-xnnpack.yml", "Qwen/Qwen3-1.7B", "sha-Qwen/Qwen3-1.7B")] + vulkan = summary["state"]["models"]["Qwen/Qwen3-1.7B"]["backends"]["vulkan"] + assert vulkan["status"] == "dispatched" and vulkan["built_with"] == "xnnpack" and "predates" in vulkan["note"] diff --git a/tools/codecheck/README.md b/tools/codecheck/README.md new file mode 100644 index 0000000..ba2126e --- /dev/null +++ b/tools/codecheck/README.md @@ -0,0 +1,18 @@ +# codecheck: is a published .pte what its report says it is? + +Byte-level checks used for the 2026-11-09 GPTQ rollout (docs/PLAN.md, Known limits; research +findings 43-45). Not wired into the pipeline or CI yet. + +- `ptecodes.py`: reads every int4 linear out of a `.pte` (ExecuTorch 1.5.1), in execution order: + XNNPACK `qbint4` filters of fully-connected nodes (delegate calls in program order), Vulkan + `et_vk.linear_dq8ca_q4gsw` weights, scales and per-group code sums. +- `codesmatch.py `: exit 0 only if every int4 linear equals the solve's codes + and scales exactly, every codes.pt entry is used, and the order is the transformer's execution + order (wq, wk, wv, wo, w1, w3, w2 per layer, head last). Ran before every publish. +- `check.py `: no codes needed; matches each int4 linear to the + fp32 source and calls it round-to-nearest or solved (thresholds calibrated on SmolLM2-135M). +- `scan.py`, `make_tasks.py`: `check.py` over every XNNPACK/Vulkan file in an org, sharded. +- `verify_hub.py`: ties every Hub file of the rolled-out repos to its byte check, codes hash, + gate reference and (Vulkan) gate proxy; per-model exceptions are listed at the top. + +Paths in `check.py` (`EXECUPACK`) and in `verify_hub.py`'s inputs assume the rollout's layout. diff --git a/tools/codecheck/check.py b/tools/codecheck/check.py new file mode 100644 index 0000000..21fb41c --- /dev/null +++ b/tools/codecheck/check.py @@ -0,0 +1,188 @@ +"""Are a .pte's int4 codes rounded to nearest, or chosen (GPTQ)? Judged from the bytes alone. + + python check.py [--codes codes.pt] [--json out.json] + +Every int4 linear in the file (XNNPACK qbint4 or Vulkan linear_dq8ca_q4gsw) is matched to the +source linear whose fp32 weights its dequantisation is nearest, then measured two ways: + +- agree_rtn: the share of its codes equal to torchao's round to nearest (scale amax/7.5, + codes clamp(round(w/scale), -8, 7)), the codes an unsolved export carries; +- scale_at_amax: the share of its groups whose scale is amax/7.5 (within bf16 rounding). + +Calibrated on SmolLM2-135M: the published RTN Vulkan file reads 0.990-0.992 and 1.0 on every +linear; the published GPTQ XNNPACK file reads 0.67-0.87 and 0.09-0.36. A linear is "rtn" at +>= 0.98 on both, "solved" below 0.95 or 0.90, and "ambiguous" otherwise. + +With --codes, every matched linear must also carry exactly the solve's codes and scales. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, __import__("os").environ.get("EXECUPACK", "/Users/alpha/execupack-gptq")) +from ptecodes import _unpack_nibbles, load_pte, vulkan_int4, xnnpack_int4 # noqa: E402 + +from pipeline import convert as pconvert # noqa: E402 +from pipeline import families + +RTN_AGREE, RTN_SCALE = 0.98, 0.98 +SOLVED_AGREE, SOLVED_SCALE = 0.95, 0.90 + + +def et_state(model_dir: Path, work: Path) -> dict: + config = json.loads((model_dir / "config.json").read_text()) + plan = families.xnnpack_plan(families.family_for(config), config) + out = work / f"{model_dir.name}-et.pth" + if not out.exists(): + pconvert.convert(model_dir, out, plan.converter, config) + sd = torch.load(out, map_location="cpu", weights_only=True) + return {k: v.float().numpy() for k, v in sd.items() if v.ndim == 2 and not k.endswith("embeddings.weight")} + + +def rtn(w, scale, group): + return np.clip(np.round(w / np.repeat(scale, group, axis=1)), -8, 7) + + +def classify(agree, at_amax): + if agree >= RTN_AGREE and at_amax >= RTN_SCALE: + return "rtn" + if agree < SOLVED_AGREE or at_amax < SOLVED_SCALE: + return "solved" + return "ambiguous" + + +def decode(t, low_first): + codes = _unpack_nibbles(t["raw"], low_first).reshape(t["dims"]).astype(np.int32) - 8 + return codes + + +def analyse(tensors, state, solved_codes=None): + by_shape = {} + for k, v in state.items(): + by_shape.setdefault(v.shape, []).append(k) + + def nearest(t, low_first, sample=64): + deq = decode(t, low_first)[:sample] * np.repeat(t["scale"][:sample], t["group"], axis=1) + return sorted( + (float(np.linalg.norm(deq - state[k][:sample]) / np.linalg.norm(state[k][:sample])), k) + for k in by_shape.get(t["dims"], []) + ) + + # Nibble order, per file: the order whose dequantisation lands nearer a source weight. + low_first = min((True, False), key=lambda lf: nearest(tensors[0], lf)[0][0]) + rows, used = [], set() + for t in tensors: + codes = decode(t, low_first) + errs = nearest(t, low_first) + (err, key), runner = errs[0], (errs[1][0] if len(errs) > 1 else float("inf")) + w = state[key] + g = t["group"] + amax = np.abs(w.reshape(w.shape[0], -1, g)).max(-1) + ratio = t["scale"] / np.maximum(amax / 7.5, 1e-12) + agree = float((codes == rtn(w, (amax / 7.5).astype(np.float32), g)).mean()) + at_amax = float((np.abs(ratio - 1) < 0.006).mean()) + deq = codes * np.repeat(t["scale"], g, axis=1) + row = { + "key": key, + "dims": list(t["dims"]), + "rel_err": float(np.linalg.norm(deq - w) / np.linalg.norm(w)), + "match_margin": float(runner / err) if err > 0 else float("inf"), + "agree_rtn": agree, + "agree_own_scale": float((codes == rtn(w, t["scale"], g)).mean()), + "scale_at_amax": at_amax, + "class": classify(agree, at_amax), + "duplicate": key in used, + } + if t.get("sums") is not None: + row["sums_ok"] = bool((codes.reshape(t["dims"][0], -1, g).sum(-1) == t["sums"]).all()) + if solved_codes is not None: + entry = solved_codes.get(key[: -len(".weight")]) + if entry is None: + row["codes_exact"] = None + else: + q = entry["qdata"].to(torch.int32).numpy() + s = entry["scale"].float().numpy() + row["codes_exact"] = bool(q.shape == codes.shape and (q == codes).all()) + row["scales_exact"] = bool(s.shape == t["scale"].shape and np.array_equal(s, t["scale"])) + rows.append(row) + used.add(key) + return low_first, rows, sorted(set(state) - used) + + +def summarise(path: Path, tensors, low_first, rows, not_int4, solved_codes): + classes = {c: sum(r["class"] == c for r in rows) for c in ("rtn", "solved", "ambiguous")} + sums_bad = any(r.get("sums_ok") is False for r in rows) + duplicates = sum(r["duplicate"] for r in rows) + if not rows: + verdict = "no-int4" + elif not_int4 or duplicates or sums_bad: + # A verdict about the codes needs every source linear found once, and a Vulkan file's + # sums consistent with its codes (codex review: a partial file must not read "gptq"). + verdict = "incomplete" + elif classes["solved"] == len(rows): + verdict = "gptq" + elif classes["rtn"] == len(rows): + verdict = "rtn" + else: + verdict = "mixed" + out = { + "file": path.name, + "backends": sorted({"vulkan" if "sums" in t else "xnnpack" for t in tensors}), + "verdict": verdict, + "int4_linears": len(rows), + "classes": classes, + "source_linears_not_int4": not_int4, + "duplicate_matches": sum(r["duplicate"] for r in rows), + "match_margin_min": min((r["match_margin"] for r in rows), default=None), + "agree_rtn_max": max((r["agree_rtn"] for r in rows), default=None), + "agree_rtn_median": float(np.median([r["agree_rtn"] for r in rows])) if rows else None, + "scale_at_amax_max": max((r["scale_at_amax"] for r in rows), default=None), + "rel_err_max": max((r["rel_err"] for r in rows), default=None), + "low_nibble_first": low_first, + } + if any("sums_ok" in r for r in rows): + out["vulkan_sums_ok"] = all(r.get("sums_ok", True) for r in rows) + if solved_codes is not None: + out["codes_exact"] = sum(r.get("codes_exact") is True for r in rows) + out["scales_exact"] = sum(r.get("scales_exact") is True for r in rows) + out["codes_missing"] = sorted(r["key"] for r in rows if r.get("codes_exact") is None) + out["codes_unused"] = sorted(set(solved_codes) - {r["key"][: -len(".weight")] for r in rows}) + return out + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument("pte") + ap.add_argument("model_dir") + ap.add_argument("work") + ap.add_argument("--codes") + ap.add_argument("--json") + ap.add_argument("--rows", action="store_true", help="print every linear") + a = ap.parse_args(argv) + path = Path(a.pte) + pte, named = load_pte(path) + tensors = xnnpack_int4(pte, named) + vulkan_int4(pte, named) + state = et_state(Path(a.model_dir), Path(a.work)) + solved = torch.load(a.codes, map_location="cpu", weights_only=True) if a.codes else None + if not tensors: + low_first, rows, not_int4 = None, [], sorted(state) + else: + low_first, rows, not_int4 = analyse(tensors, state, solved) + out = summarise(path, tensors, low_first, rows, not_int4, solved) + if a.rows: + out["rows"] = rows + text = json.dumps(out, indent=1) + if a.json: + Path(a.json).write_text(text) + print(text) + + +if __name__ == "__main__": + main() diff --git a/tools/codecheck/codesmatch.py b/tools/codecheck/codesmatch.py new file mode 100644 index 0000000..df1d1c5 --- /dev/null +++ b/tools/codecheck/codesmatch.py @@ -0,0 +1,165 @@ +"""Does a built .pte carry exactly the solve's codes? Read from the file's bytes, before publish. + + python codesmatch.py [--allow-unused output] + +Every int4 linear in the file (XNNPACK qbint4 or Vulkan linear_dq8ca_q4gsw) must equal one +entry of codes.pt exactly: the same int4 codes and the same scales (bf16 in an XNNPACK file, +fp32 in a Vulkan one; the solve picks scales bf16 can hold, so both are exact). Every codes.pt +entry must be used once, except names given with --allow-unused (an int8 output head has no +int4 codes to carry). Matching is by content, so no fp32 source and no layer names are needed. + +Exit 0 when the file carries the codes and nothing else at int4; 1 otherwise, with the reasons. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ptecodes import _unpack_nibbles, load_pte, vulkan_int4, xnnpack_int4 # noqa: E402 + +# Execution order inside one transformer layer of ExecuTorch's llama_transformer (attention +# wq, wk, wv, then wo; feed-forward w2(silu(w1(x)) * w3(x))). Both delegates serialize their +# int4 linears in this order, layers ascending and the head last, so a file whose matched +# names follow it carries each module's codes where that module's computation reads them. +LAYER_ORDER = [ + "attention.wq", + "attention.wk", + "attention.wv", + "attention.wo", + "feed_forward.w1", + "feed_forward.w3", + "feed_forward.w2", +] + + +def expected_order(names): + """codes.pt names in execution order, or None if a name falls outside the known layout.""" + import re + + def key(name): + if name == "output": + return (10**9, 0) + m = re.fullmatch(r"layers\.(\d+)\.(.+)", name) + if not m or m.group(2) not in LAYER_ORDER: + raise KeyError(name) + return (int(m.group(1)), LAYER_ORDER.index(m.group(2))) + + try: + return sorted(names, key=key) + except KeyError: + return None + + +def digest(codes: np.ndarray) -> str: + c = np.ascontiguousarray(codes.astype(np.int8)) + return hashlib.sha256(repr(c.shape).encode() + c.tobytes()).hexdigest() + + +def bf16(x: np.ndarray) -> np.ndarray: + return torch.from_numpy(np.ascontiguousarray(x)).to(torch.bfloat16).float().numpy() + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("pte") + ap.add_argument("codes") + ap.add_argument("--allow-unused", action="append", default=[]) + a = ap.parse_args(argv) + + solved = torch.load(a.codes, map_location="cpu", weights_only=True) + index = {} + for name, entry in solved.items(): + index.setdefault(digest(entry["qdata"].to(torch.int8).numpy()), []).append(name) + + pte, named = load_pte(Path(a.pte)) + xnn, vk = xnnpack_int4(pte, named), vulkan_int4(pte, named) + tensors = xnn + vk + problems, used, matched = [], {}, 0 + if not tensors: + problems.append("the file has no int4 linear") + # Nibble order is a property of the backend's packing: decide it once, on whichever order + # finds the first tensor in codes.pt. + low_first = True + if tensors: + first = tensors[0] + hits = { + lf: digest(_unpack_nibbles(first["raw"], lf).reshape(first["dims"]).astype(np.int16) - 8) in index + for lf in (True, False) + } + low_first = hits[True] or not hits[False] + # The names the file should hold, in execution order: a tensor whose codes appear under + # several names takes the one expected at its position (codex review, round 2). + want_order = expected_order([n for n in solved if n not in a.allow_unused]) + for i, t in enumerate(tensors): + codes = _unpack_nibbles(t["raw"], low_first).reshape(t["dims"]).astype(np.int16) - 8 + names = [n for n in index.get(digest(codes), []) if n not in used] + if not names: + problems.append(f"int4 tensor {i} {t['dims']} matches no unused codes.pt entry") + continue + have = t["scale"] + + def stored(n, t=t): + w = solved[n]["scale"].float().numpy() + return bf16(w) if "sums" not in t else w + + exact = [n for n in names if stored(n).shape == have.shape and np.array_equal(stored(n), have)] + expected_here = want_order[i] if want_order and i < len(want_order) else None + pool = exact or names + name = expected_here if expected_here in pool else pool[0] + if not exact: + want = stored(name) + diff = float(np.abs(want - have).max()) if want.shape == have.shape else None + problems.append(f"{name}: scales differ (max abs diff {diff})") + if t.get("sums") is not None and not (codes.reshape(t["dims"][0], -1, t["group"]).sum(-1) == t["sums"]).all(): + problems.append(f"{name}: Vulkan per-group code sums disagree with the codes") + used[name] = i + matched += 1 + order = [n for n, _ in sorted(used.items(), key=lambda kv: kv[1])] + if a.allow_unused: + want_order = expected_order([n for n in solved if n not in a.allow_unused or n in used]) + if want_order is None: + problems.append("codes.pt has a module outside the known layer layout; order not checkable") + elif order != want_order: + first = next( + (k for k, (x, y) in enumerate(zip(order, want_order, strict=False)) if x != y), + min(len(order), len(want_order)), + ) + problems.append( + f"linears are not in execution order: position {first} holds " + f"{order[first] if first < len(order) else None}, expected " + f"{want_order[first] if first < len(want_order) else None}" + ) + unused = sorted(set(solved) - set(used) - set(a.allow_unused)) + if unused: + problems.append(f"codes.pt entries not in the file: {unused[:8]}{' ...' if len(unused) > 8 else ''}") + sha = hashlib.sha256() + with open(a.pte, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + sha.update(chunk) + result = { + "file": Path(a.pte).name, + "sha256": sha.hexdigest(), + "backend": "vulkan" if vk else "xnnpack", + "int4_linears": len(tensors), + "matched_exactly": matched, + "codes_entries": len(solved), + "unused_allowed": sorted(set(a.allow_unused) & (set(solved) - set(used))), + "low_nibble_first": low_first, + "order_checked": want_order is not None, + "ok": not problems, + "problems": problems, + } + print(json.dumps(result, indent=1)) + return 0 if not problems else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/codecheck/make_tasks.py b/tools/codecheck/make_tasks.py new file mode 100644 index 0000000..e76d657 --- /dev/null +++ b/tools/codecheck/make_tasks.py @@ -0,0 +1,41 @@ +"""Every XNNPACK and Vulkan .pte in the org, split into N shards by repo (a shard keeps one +source model's state in memory at a time), for parallel scan.py workers. + + python make_tasks.py [--only REGEX | --exclude REGEX] +""" + +import json +import re +import sys +from pathlib import Path + +from huggingface_hub import HfApi + +org, n, out = sys.argv[1], int(sys.argv[2]), Path(sys.argv[3]) +only = re.compile(sys.argv[sys.argv.index("--only") + 1]) if "--only" in sys.argv else None +exclude = re.compile(sys.argv[sys.argv.index("--exclude") + 1]) if "--exclude" in sys.argv else None +out.mkdir(parents=True, exist_ok=True) +api = HfApi() +repos = [] +for m in api.list_models(author=org): + if (only and not only.search(m.id)) or (exclude and exclude.search(m.id)): + continue + info = api.model_info(m.id, files_metadata=True) + ptes = sorted( + s.rfilename + for s in info.siblings + if s.rfilename.endswith(".pte") and s.rfilename.split("/")[0] in ("xnnpack", "vulkan") + ) + if ptes: + size = sum(s.size or 0 for s in info.siblings if s.rfilename in ptes) + repos.append((size, m.id, ptes)) +repos.sort(reverse=True) +shards = [[] for _ in range(n)] +load = [0] * n +for size, repo, ptes in repos: # biggest first onto the lightest shard + i = load.index(min(load)) + shards[i] += [[repo, p] for p in ptes] + load[i] += size +for i, s in enumerate(shards): + (out / f"tasks-{i}.json").write_text(json.dumps(s, indent=0)) +print(len(repos), "repos,", sum(len(s) for s in shards), "files,", [f"{x / 1e9:.0f} GB" for x in load]) diff --git a/tools/codecheck/ptecodes.py b/tools/codecheck/ptecodes.py new file mode 100644 index 0000000..68505ec --- /dev/null +++ b/tools/codecheck/ptecodes.py @@ -0,0 +1,182 @@ +"""Pull the int4 linears (codes and scales) out of an ExecuTorch .pte, for XNNPACK and Vulkan. + +Reads only the file: no runtime, no reports. Each linear comes back as +{"codes": int8 [out, in] in -8..7, "scale": float32 [out, in // group]} so it can be compared +with the fp32 source weights. +""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +from pathlib import Path + +import executorch +import numpy as np +from executorch.exir._serialize._flatbuffer import _get_flatc_path +from executorch.exir._serialize._program import deserialize_pte_binary + +ET = Path(list(executorch.__path__)[0]) + + +def _flatc_json(schema: Path, blob: bytes) -> dict: + with tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "g.bin" + bin_path.write_bytes(blob) + subprocess.run( + [_get_flatc_path(), "--json", "--strict-json", "--raw-binary", "-o", tmp, str(schema), "--", str(bin_path)], + check=True, + capture_output=True, + ) + return json.loads((Path(tmp) / "g.json").read_text()) + + +def load_pte(path: Path): + data = Path(path).read_bytes() + pte = deserialize_pte_binary(data) + named = {} + if pte.named_data is not None: + store = pte.named_data + for key, entry in store.pte_data.items(): + named[key] = bytes(store.buffers[entry.buffer_index]) + return pte, named + + +def delegates(pte): + """Delegate payloads in the order the program calls them (its DelegateCall instructions), + not the order of the delegates table: position in this list is position in execution.""" + from executorch.exir.schema import DelegateCall + + program = pte.program + out = [] + for plan in program.execution_plan: + for chain in plan.chains: + for instr in chain.instructions: + call = instr.instr_args + if not isinstance(call, DelegateCall): + continue + d = plan.delegates[call.delegate_index] + loc = d.processed + if loc.location.name != "INLINE": + raise RuntimeError(f"segment delegate data not restored for {d.id}") + out.append((plan.name, d.id, bytes(program.backend_delegate_data[loc.index].data))) + return out + + +if __name__ == "__main__": + import sys + + pte, named = load_pte(Path(sys.argv[1])) + print("named data keys:", len(named), list(named)[:3]) + for plan, did, blob in delegates(pte): + print(plan, did, len(blob), blob[4:8]) + + +def _bf16_to_f32(raw: bytes) -> np.ndarray: + u16 = np.frombuffer(raw, dtype=" np.ndarray: + b = np.frombuffer(raw, dtype=np.uint8) + lo, hi = (b & 0x0F).astype(np.int16), (b >> 4).astype(np.int16) + pair = (lo, hi) if low_first else (hi, lo) + return np.stack(pair, axis=-1).reshape(-1) + + +def xnnpack_int4(pte, named) -> list[dict]: + """The qbint4 weight of every XNNPACK fully-connected node, in execution order: delegate + calls in program order, nodes in graph order, each node's filter resolved by value id. + A qbint4 value that no fully-connected node uses is an error, not a silent extra.""" + from executorch.backends.xnnpack.serialization.xnnpack_graph_serialize import XNNHeader + + schema = ET / "backends/xnnpack/serialization/schema.fbs" + out = [] + for plan, did, blob in delegates(pte): + if did != "XnnpackBackend": + continue + h = XNNHeader.from_bytes(blob[: XNNHeader.EXPECTED_LENGTH]) + g = _flatc_json(schema, blob[h.flatbuffer_offset : h.flatbuffer_offset + h.flatbuffer_size]) + consts = g.get("constant_data", []) + + def read(idx, consts=consts, h=h, blob=blob): + c = consts[idx] + if c.get("named_key"): + return named[c["named_key"]] + start = h.constant_data_offset + int(c["offset"]) + return blob[start : start + int(c["size"])] + + by_id, int4_ids = {}, set() + for v in g.get("xvalues", []): + u = v.get("xvalue_union", {}) + tv = u.get("tensor_value", u) + if "id_out" in tv: + by_id[tv["id_out"]] = v + if v.get("xvalue_union_type") == "XNNQuantizedTensorValue" and tv.get("datatype") == "xnn_datatype_qbint4": + int4_ids.add(tv["id_out"]) + used = [] + for node in g.get("xnodes", []): + if node.get("xnode_union_type") != "XNNFullyConnected": + continue + fid = node["xnode_union"]["filter_id"] + if fid not in int4_ids: + continue + u = by_id[fid]["xvalue_union"] + tv, qp = u["tensor_value"], u["quant_params"] + if u.get("quant_params_type") != "PerChannelGroupQuant": + raise RuntimeError(f"qbint4 filter {fid} without per-channel-group quantization") + rows, cols = tv["dims"] + group = qp["group_size"] + raw = read(tv["constant_buffer_idx"]) + assert len(raw) * 2 == rows * cols, (len(raw), rows, cols) + scale = _bf16_to_f32(read(qp["scale_buffer_idx"])).reshape(rows, cols // group) + out.append({"plan": plan, "dims": (rows, cols), "group": group, "raw": raw, "scale": scale}) + used.append(fid) + if sorted(used) != sorted(int4_ids): + raise RuntimeError(f"XNNPACK delegate: int4 values {sorted(int4_ids)} used by FC nodes as {sorted(used)}") + return out + + +def vulkan_int4(pte, named) -> list[dict]: + """Every et_vk.linear_dq8ca_q4gsw weight: packed nibbles [out, in/2], fp32 scales [groups, out] + (returned transposed to [out, groups]), and the int32 per-group code sums the shader uses.""" + from executorch.backends.vulkan.serialization.vulkan_graph_serialize import VulkanDelegateHeader as H + + schema = ET / "backends/vulkan/serialization/schema.fbs" + out = [] + for plan, did, blob in delegates(pte): + if did != "VulkanBackend": + continue + h = H.from_bytes(blob[: H.EXPECTED_LENGTH]) + g = _flatc_json(schema, blob[h.flatbuffer_offset : h.flatbuffer_offset + h.flatbuffer_size]) + vals, consts = g["values"], g.get("constants", []) + + def tensor(i, vals=vals, consts=consts, blob=blob, h=h): + v = vals[i]["value"] + c = consts[v["constant_id"]] + raw = ( + named[c["named_key"]] + if c.get("named_key") + else blob[h.bytes_offset + int(c["offset"]) :][: int(c["length"])] + ) + return v, raw + + for call in g["chain"]: + if call["name"] != "et_vk.linear_dq8ca_q4gsw.default": + continue + a = call["args"] + wv, wraw = tensor(a[3]) + # One linear (the head, in SmolLM2) computes its sums at run time: no constant to read. + has_sums = vals[a[4]]["value"].get("constant_id", -1) >= 0 + sv, sraw = tensor(a[4]) if has_sums else (None, None) + cv, craw = tensor(a[5]) + group = vals[a[6]]["value"]["int_val"] + rows, half = wv["dims"] + assert wv["datatype"] == "UINT8" and cv["datatype"] == "FLOAT32" + scale = np.frombuffer(craw, dtype=" 5 else {} +tasks = json.loads(tasks_path.read_text()) +done = set() +if results_path.exists(): + for line in results_path.read_text().splitlines(): + r = json.loads(line) + done.add((r["repo"], r["path"])) +api = HfApi() +lfs = {} +state_cache = {} + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + h.update(chunk) + return h.hexdigest() + + +for repo, path in tasks: + if (repo, path) in done: + continue + result = {"repo": repo, "path": path} + try: + if repo not in lfs: + info = api.model_info(repo, files_metadata=True) + lfs[repo] = { + "sha": info.sha, + "files": {s.rfilename: (s.lfs.sha256 if s.lfs else None) for s in info.siblings}, + } + result["repo_sha"] = lfs[repo]["sha"] + folder, name = path.rsplit("/", 1) + window = name.rsplit("-", 1)[1].removesuffix(".pte") + report = json.loads( + Path(hf_hub_download(repo, f"{folder}/export-report-{window}.json", revision=lfs[repo]["sha"])).read_text() + ) + rfile = next(f for f in report["files"] if f["path"] == path) + recipe = report.get("recipe", {}) + result.update( + claimed_codes=recipe.get("int4_codes"), + claimed_label=recipe.get("label"), + executorch=(report.get("toolchain") or {}).get("executorch"), + gate_passed=(report.get("gate") or {}).get("passed") if isinstance(report.get("gate"), dict) else None, + gate_proxy=bool(report.get("gate_proxy")), + source=report["source"]["id"], + source_sha=report["source"]["sha"], + ) + local = Path(hf_hub_download(repo, path, revision=lfs[repo]["sha"], local_dir=dl / "pte")) + digest = sha256(local) + result.update( + sha256=digest, + sha_matches_report=digest == rfile["sha256"], + sha_matches_hub=digest == lfs[repo]["files"].get(path), + bytes=local.stat().st_size, + bytes_match_report=local.stat().st_size == rfile["bytes"], + ) + src_key = (report["source"]["id"], report["source"]["sha"]) + if repo in overrides: + src_key = tuple(overrides[repo]) + result["compared_with"] = list(src_key) + src_dir = dl / "src" / f"{src_key[0].replace('/', '--')}@{src_key[1][:12]}" + if not (src_dir / "config.json").exists(): + snapshot_download( + src_key[0], revision=src_key[1], allow_patterns=["*.safetensors", "*.json"], local_dir=src_dir + ) + if src_key not in state_cache: + state_cache.clear() + state_cache[src_key] = check.et_state(src_dir, work) + state = state_cache[src_key] + pte, named = load_pte(local) + tensors = xnnpack_int4(pte, named) + vulkan_int4(pte, named) + if tensors: + low_first, rows, not_int4 = check.analyse(tensors, state) + else: + low_first, rows, not_int4 = None, [], sorted(state) + result.update(check.summarise(local, tensors, low_first, rows, not_int4, None)) + if repo in overrides: + result["rows"] = [ + {k: r[k] for k in ("key", "class", "rel_err", "agree_rtn", "scale_at_amax")} for r in rows + ] + del pte, named, tensors + local.unlink() + except Exception as error: # record and move on: one bad file must not stop the scan + result["error"] = f"{type(error).__name__}: {error}" + result["traceback"] = traceback.format_exc()[-2000:] + with open(results_path, "a") as f: + f.write(json.dumps(result) + "\n") + print(repo, path, result.get("verdict"), result.get("error", ""), flush=True) +shutil.rmtree(dl / "pte", ignore_errors=True) +print("SCAN DONE", flush=True) diff --git a/tools/codecheck/verify_hub.py b/tools/codecheck/verify_hub.py new file mode 100644 index 0000000..78b0f5d --- /dev/null +++ b/tools/codecheck/verify_hub.py @@ -0,0 +1,276 @@ +"""After the rollout: is every file in the rolled-out repos what its report says it is? + + python verify_hub.py [--skip MODEL ...] + +Per repo, from the Hub, against the byte-check records collected from the boxes +(//gptq-/codesmatch-*.json and codes.sha256): +- xnnpack/ and vulkan/ each hold exactly the five windows: one report per window label, and the + .pte files the reports name are exactly the .pte files in the folder, one each; +- every report: int4_codes gptq, ExecuTorch 1.5.1, the pinned source revision, a file sha256 + equal to the Hub's LFS sha256, and a measured, passing gate (XNNPACK: graded > 0, fp32 and + export means present, no override) or a Vulkan gate proxy whose verdict equals the published + XNNPACK report's of the same window and whose sha256 is that XNNPACK file; +- the report's codes_sha256 is the hash of this model's codes.pt as it was on the boxes, and + the .pte was byte-checked against that codes.pt (an ok codesmatch record in this model's own + runs directory with the file's sha256); +- README.md and vulkan/config.json carry no round-to-nearest claim. +Exit 1 if any repo has a problem. +""" + +import hashlib +import json +import sys +from pathlib import Path + +from huggingface_hub import HfApi, hf_hub_download + +WINDOWS = {"2k": 2048, "4k": 4096, "8k": 8192, "16k": 16384, "32k": 32768} +GATE_FIELDS = ("passed", "graded", "agreed", "fp32_mean", "export_mean") + +args = sys.argv[1:] +skip = set(args[args.index("--skip") + 1 :]) if "--skip" in args else set() +models = [m for m in json.loads(Path(args[0]).read_text()) if m["model"] not in skip] +collected, out_path = Path(args[1]), Path(args[2]) + + +def slug(model_id): + return model_id.split("/")[1].lower().replace(".", "") + + +# Decided exceptions (user, 2026-11-09): Qwen3-0.6B ships fp32 linears on XNNPACK and no Vulkan +# folder; SmolLM2-360M keeps its fp32 XNNPACK files and ships GPTQ Vulkan files whose gate proxy +# is an XNNPACK twin that was gated but not published (its reports are kept as evidence). +FP32_REBUILT = {"Qwen/Qwen3-0.6B"} +FP32_KEPT = {"HuggingFaceTB/SmolLM2-360M-Instruct"} +NO_VULKAN = {"Qwen/Qwen3-0.6B"} + +evidence = {} +for runs_dir in collected.glob("*/gptq-*"): + ev = evidence.setdefault( + runs_dir.name.removeprefix("gptq-"), {"codes": set(), "checked": {}, "gate_ref": set(), "twins": {}} + ) + for t in (runs_dir / "twin-reports").glob("export-report-*.json"): + d = json.loads(t.read_text()) + ev["twins"][d["files"][0]["sha256"]] = d + if (runs_dir / "gate.json").exists(): + ev["gate_ref"].add(hashlib.sha256((runs_dir / "gate.json").read_bytes()).hexdigest()) + if (runs_dir / "codes.sha256").exists(): + ev["codes"].add((runs_dir / "codes.sha256").read_text().strip()) + for f in runs_dir.glob("codesmatch-*.json"): + try: + d = json.loads(f.read_text()) + except ValueError: + continue + fp32_record = f.name.startswith("codesmatch-xnnpack-fp32-") + sound = d.get("ok") is True and ( + (fp32_record and d.get("int4_linears") == 0) + or ( + not fp32_record + and d.get("int4_linears", 0) > 0 + and d.get("matched_exactly") == d.get("int4_linears") + and d.get("order_checked") is True + and f.name.startswith(f"codesmatch-{d.get('backend')}-") + ) + ) + if sound: + ev["checked"][d["sha256"]] = { + "file": d["file"], + "matched": d["matched_exactly"], + "of": d["int4_linears"], + "backend": "xnnpack" if fp32_record else d.get("backend"), + "record": str(f), + } + +api = HfApi() +out, failed = [], False +for m in models: + repo, rev_src = m["repo"], m["main_sha"] + ev = evidence.get(slug(m["model"]), {"codes": set(), "checked": {}, "gate_ref": set(), "twins": {}}) + fp32_rebuilt, fp32_kept, no_vulkan = m["model"] in FP32_REBUILT, m["model"] in FP32_KEPT, m["model"] in NO_VULKAN + info = api.model_info(repo, files_metadata=True) + files = {s.rfilename: s for s in info.siblings} + problems, rows = [], {} + if len(ev["codes"]) != 1 and not fp32_rebuilt: + problems.append( + f"codes.pt hash for {slug(m['model'])}: {sorted(ev['codes']) or 'none collected'}, expected exactly one" + ) + codes_sha = next(iter(ev["codes"])) if len(ev["codes"]) == 1 else None + for folder in ("xnnpack", "vulkan"): + if folder == "vulkan" and no_vulkan: + left = sorted(f for f in files if f.startswith("vulkan/")) + if left: + problems.append(f"vulkan/ should be withdrawn, still holds {left}") + continue + fp32 = folder == "xnnpack" and (fp32_rebuilt or fp32_kept) + ptes = sorted(f for f in files if f.startswith(folder + "/") and f.endswith(".pte")) + reports = sorted(f for f in files if f.startswith(folder + "/export-report-")) + labels = [r.rsplit("-", 1)[1].removesuffix(".json") for r in reports] + if sorted(labels) != sorted(WINDOWS): + problems.append(f"{folder}: report labels {labels}, expected exactly {list(WINDOWS)}") + named = [] + for r, label in zip(reports, labels, strict=True): + d = json.loads(Path(hf_hub_download(repo, r, revision=info.sha)).read_text()) + rec, f0 = d["recipe"], d["files"][0] + if len(d["files"]) != 1: + problems.append(f"{r}: names {len(d['files'])} files") + named.append(f0["path"]) + lfs = files.get(f0["path"]) + gate = d.get("gate") if folder == "xnnpack" else (d.get("gate_proxy") or {}).get("gate") + row = { + "report": r, + "file": f0["path"], + "sha256": f0["sha256"], + "codes_sha256": rec.get("codes_sha256"), + "gate": {k: (gate or {}).get(k) for k in GATE_FIELDS}, + } + rows[(folder, label)] = row + if d["window"]["context"] != WINDOWS.get(label): + problems.append(f"{r}: window {d['window']['context']} under label {label}") + kind = "fp32" if fp32 else "gptq" + if ( + not f0["path"].startswith(folder + "/") + or f"-{kind}-" not in f0["path"] + or not f0["path"].endswith(f"-{label}.pte") + ): + problems.append(f"{r}: file name {f0['path']} is not this folder's {kind} {label} file") + if lfs is None or lfs.lfs is None or lfs.lfs.sha256 != f0["sha256"]: + problems.append( + f"{r}: names {f0['path']} sha {f0['sha256'][:12]}, " + f"Hub has {getattr(getattr(lfs, 'lfs', None), 'sha256', None)}" + ) + if d["source"]["sha"] != rev_src: + problems.append(f"{r}: source revision {d['source']['sha'][:12]} != {rev_src[:12]}") + if fp32: + if rec.get("int4_codes") is not None or rec.get("qmode") != "fp32": + problems.append( + f"{r}: expected fp32 linears, recipe says {rec.get('qmode')} / {rec.get('int4_codes')}" + ) + if fp32_kept: + continue # the deliberate fp32 files built on 1.4.0, untouched by this rollout + elif rec.get("int4_codes") != "gptq": + problems.append(f"{r}: int4_codes {rec.get('int4_codes')}") + if d["toolchain"]["executorch"] != "1.5.1": + problems.append(f"{r}: ExecuTorch {d['toolchain']['executorch']}") + measured = ( + (gate or {}).get("passed") is True + and not (gate or {}).get("override") + and isinstance((gate or {}).get("graded"), int) + and gate["graded"] > 0 + and all((gate or {}).get(k) is not None for k in ("agreed", "fp32_mean", "export_mean")) + ) + if not measured: + problems.append(f"{r}: gate {'proxy ' if folder == 'vulkan' else ''}is not a measured pass: {gate}") + if folder == "vulkan": + proxy = d.get("gate_proxy") or {} + row["proxy_sha256"] = proxy.get("sha256") + row["proxy_file"], row["proxy_report"] = proxy.get("file"), proxy.get("report") + elif len(ev["gate_ref"]) != 1 or (gate or {}).get("reference_sha256") not in ev["gate_ref"]: + # The verdict must have been measured against this model's own fp32 reference. + problems.append( + f"{r}: gate measured against {str((gate or {}).get('reference_sha256'))[:12]}, " + f"this model's gate.json is {sorted(ev['gate_ref']) or 'not collected'}" + ) + chk = ev["checked"].get(f0["sha256"]) + if chk and (chk["file"] != f0["path"].split("/")[-1] or chk["backend"] != folder): + problems.append(f"{r}: byte-check record {chk['record']} is for {chk['backend']}/{chk['file']}") + if fp32: + if f0["sha256"] not in ev["checked"]: + problems.append(f"{r}: {f0['path']} sha {f0['sha256'][:12]} has no ok fp32 byte check") + else: + row["byte_check"] = ev["checked"][f0["sha256"]] + continue + if not codes_sha or rec.get("codes_sha256") != codes_sha: + problems.append( + f"{r}: codes_sha256 {str(rec.get('codes_sha256'))[:12]} " + f"is not this model's codes.pt {str(codes_sha)[:12]}" + ) + if f0["sha256"] not in ev["checked"]: + problems.append(f"{r}: {f0['path']} sha {f0['sha256'][:12]} has no ok byte check in this model's runs") + else: + row["byte_check"] = ev["checked"][f0["sha256"]] + if sorted(named) != ptes or len(set(named)) != len(named): + problems.append(f"{folder}: reports name {sorted(named)}, the folder holds {ptes}") + cfg_doc = json.loads(Path(hf_hub_download(repo, f"{folder}/config.json", revision=info.sha)).read_text()) + want = sorted((v["file"].split("/")[-1], v["sha256"], WINDOWS[k[1]]) for k, v in rows.items() if k[0] == folder) + have = sorted((v["file"], v["sha256"], v["context"]) for v in cfg_doc.get("variants", [])) + if want != have: + problems.append(f"{folder}/config.json variants {have} differ from the reports {want}") + for v in cfg_doc.get("variants", []): + lfs = files.get(f"{folder}/{v['file']}") + if lfs is None or lfs.size != v.get("size_bytes"): + problems.append( + f"{folder}/config.json: {v['file']} size {v.get('size_bytes')} != Hub {getattr(lfs, 'size', None)}" + ) + for label in WINDOWS: + x, v = rows.get(("xnnpack", label)), rows.get(("vulkan", label)) + if v and fp32_kept: + twin = ev["twins"].get(v.get("proxy_sha256")) + if not twin: + problems.append( + f"vulkan {label}: proxy sha {str(v.get('proxy_sha256'))[:12]} matches no kept twin report" + ) + continue + tg = {k: (twin.get("gate") or {}).get(k) for k in GATE_FIELDS} + if ( + tg != v["gate"] + or twin["recipe"].get("codes_sha256") != v["codes_sha256"] + or twin["window"]["context"] != WINDOWS[label] + ): + problems.append(f"vulkan {label}: proxy does not match its kept twin report") + if twin["files"][0]["sha256"] not in ev["checked"] or v.get("proxy_file") != twin["files"][0]["path"]: + problems.append( + f"vulkan {label}: kept twin {twin['files'][0]['path']} " + "lacks a byte check or is not the file the proxy names" + ) + tgate = twin.get("gate") or {} + if not ( + tgate.get("passed") is True + and not tgate.get("override") + and (tgate.get("graded") or 0) > 0 + and tgate.get("reference_sha256") in ev["gate_ref"] + and len(ev["gate_ref"]) == 1 + ): + problems.append(f"vulkan {label}: kept twin is not a measured pass against this model's gate.json") + continue + if not (x and v): + continue + if v.get("proxy_file") != x["file"] or v.get("proxy_report") != x["report"].split("/")[-1]: + problems.append( + f"vulkan {label}: proxy names {v.get('proxy_file')} / {v.get('proxy_report')}, the twin is {x['file']}" + ) + if v.get("proxy_sha256") != x["sha256"]: + problems.append( + f"vulkan {label}: proxy sha {str(v.get('proxy_sha256'))[:12]} " + f"is not the published xnnpack file {x['sha256'][:12]}" + ) + if v["gate"] != x["gate"]: + problems.append(f"vulkan {label}: proxy verdict {v['gate']} differs from the xnnpack report's {x['gate']}") + if v["codes_sha256"] != x["codes_sha256"]: + problems.append(f"{label}: xnnpack and vulkan name different codes") + readme = Path(hf_hub_download(repo, "README.md", revision=info.sha)).read_text() + cfg = "" if no_vulkan else Path(hf_hub_download(repo, "vulkan/config.json", revision=info.sha)).read_text() + if "rounded to nearest" in cfg or "round-to-nearest" in cfg: + problems.append("vulkan/config.json still says rounded to nearest") + import re + + for link in sorted(set(re.findall(r"(?:xnnpack|vulkan)/[A-Za-z0-9._-]+\.pte", readme))): + if link not in files: + problems.append(f"README links {link}, which is not in the repo") + for ln in readme.splitlines(): + if "rounded to nearest" in ln or "round-to-nearest" in ln: + problems.append(f"README: {ln.strip()[:200]}") + failed |= bool(problems) + out.append( + { + "repo": repo, + "sha": info.sha, + "ok": not problems, + "problems": problems, + "rows": {f"{k[0]}/{k[1]}": v for k, v in rows.items()}, + } + ) + print(repo, "OK" if not problems else f"{len(problems)} problems", flush=True) + for p in problems: + print(" ", p, flush=True) +out_path.write_text(json.dumps(out, indent=1)) +sys.exit(1 if failed else 0)