From 7ce7e7355acfe221c54e95e2a54df30aba5a9246 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 08:46:47 +0800 Subject: [PATCH 01/10] Phi-4-mini on CPU and GPU, and a decision gate that grades a reasoning-only template Phi-4-mini-instruct (Phi3ForCausalLM) gets a recipe for export_llm's phi_4_mini class: - The fused qkv_proj and gate_up_proj are split where HF splits them. - LongRoPE's factors pass through as ExecuTorch 1.5.1 reads them. - ffn_dim_multiplier is dropped. Upstream's config sets it to 1, which makes Rope build its table at twice the window, so a 4k export would run every prompt on the long factors that HF only uses past 4096 tokens. - Outputs are named Phi-4-mini-instruct-3.8B, because the app reads the size from the name and the release has none. - Its chat template reads tools from the system message, not from tools=, so calibration and the gate now put the schema there, as the app does. Found by codex review: before this, the Phi rows carried no tools at all. The gate closes a think block that a template leaves open. Qwen3-4B-Thinking-2507 opens one whatever enable_thinking says, which put the first word of the reasoning in the graded position. Every published family renders exactly as before. Co-Authored-By: Claude Opus 5.5 --- pipeline/calibration.py | 23 ++- pipeline/convert.py | 49 +++++- pipeline/eligibility.py | 4 + pipeline/families.py | 69 +++++++++ pipeline/gate.py | 16 +- pipeline/naming.py | 10 +- tests/conftest.py | 2 + tests/fixtures/et-phi_4_mini.params.json | 19 +++ .../fixtures/phi-4-mini-instruct.config.json | 144 ++++++++++++++++++ tests/test_convert.py | 39 +++++ tests/test_eligibility.py | 13 ++ tests/test_families.py | 52 +++++++ tests/test_gate.py | 61 ++++++++ tests/test_naming.py | 10 ++ 14 files changed, 505 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/et-phi_4_mini.params.json create mode 100644 tests/fixtures/phi-4-mini-instruct.config.json diff --git a/pipeline/calibration.py b/pipeline/calibration.py index c2333c7..5422503 100644 --- a/pipeline/calibration.py +++ b/pipeline/calibration.py @@ -42,6 +42,7 @@ from __future__ import annotations +import json from pathlib import Path # Kept short on purpose: every prompt costs an fp32 generation pass on the runner, and the @@ -273,9 +274,27 @@ def render_app(tokenizer, user_turn: str, thinking: bool = True) -> str: ignore), so the next token is the answer or the tool call itself. Calibration leaves it at the template's default and renders exactly as before. """ - messages = [{"role": "system", "content": APP_SYSTEM}, *APP_PRIMER, {"role": "user", "content": user_turn}] + system = {"role": "system", "content": APP_SYSTEM} extra = {} if thinking else {"enable_thinking": False} - return tokenizer.apply_chat_template(messages, tools=APP_TOOLS, tokenize=False, add_generation_prompt=True, **extra) + if system_carries_tools(tokenizer): + system["tools"] = json.dumps(APP_TOOLS) + else: + extra["tools"] = APP_TOOLS + messages = [system, *APP_PRIMER, {"role": "user", "content": user_turn}] + return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, **extra) + + +def system_carries_tools(tokenizer) -> bool: + """Whether the template reads its tools from the system message rather than ``tools=``. + + Phi-4-mini's does: it prints a system message's ``tools`` field, a JSON string, between + ``<|tool|>`` markers, and never reads the ``tools`` argument, so passing the schema the + usual way renders a prompt with no tools in it at all (codex review, 2026-11-09). The + app sends exactly that field: ``json.dumps`` of the list, ``", "`` between tools + (Phi4Prompt.render, ToolDefinition.asToolJson). + """ + template = getattr(tokenizer, "chat_template", None) + return isinstance(template, str) and "'tools' in message" in template def app_head(tokenizer) -> str: diff --git a/pipeline/convert.py b/pipeline/convert.py index 9b165fb..cbc8119 100644 --- a/pipeline/convert.py +++ b/pipeline/convert.py @@ -95,6 +95,24 @@ # as B, C and x stacked on dim 0, and ExecuTorch's ShortConv reads them apart. _IN_PROJ = ("conv.in_proj.weight", ("conv.B_proj.weight", "conv.C_proj.weight", "conv.x_proj.weight")) SPLIT_THREE = {"lfm2": _IN_PROJ, "lfm3": _IN_PROJ} +# Phi-4-mini (Phi3ForCausalLM) fuses q, k and v into one projection and the gate and up +# projections into another; everything else has Llama's names. HF splits them in forward: +# q, k, v at n_heads * head_dim and n_kv_heads * head_dim rows (Phi3Attention), gate before +# up in halves (Phi3MLP, gate_up_states.chunk(2, dim=-1)), as ExecuTorch's own +# examples/models/phi_4_mini/convert_weights.py does. The placeholders below never reach the +# checkpoint: convert() replaces each with its parts. +_PHI4_LAYER_KEYS = { + "self_attn.qkv_proj.weight": "attention.wqkv.weight", + "self_attn.o_proj.weight": "attention.wo.weight", + "input_layernorm.weight": "attention_norm.weight", + "post_attention_layernorm.weight": "ffn_norm.weight", + "mlp.gate_up_proj.weight": "feed_forward.w13.weight", + "mlp.down_proj.weight": "feed_forward.w2.weight", +} +_PHI4_FUSED = { + "attention.wqkv.weight": ("attention.wq.weight", "attention.wk.weight", "attention.wv.weight"), + "feed_forward.w13.weight": ("feed_forward.w1.weight", "feed_forward.w3.weight"), +} # Buffers some checkpoints carry that the model recomputes. _IGNORED = re.compile(r"(rotary_emb\.inv_freq|\.masked_bias|\.attn\.bias)$") @@ -107,9 +125,11 @@ # common one; every layer is a short convolution or an attention block, never both. "lfm2": (_LFM2_LAYER_KEYS, False), "lfm3": (_LFM3_LAYER_KEYS, False), + # HF RoPE (use_hf_rope), so q and k stay in HF's layout. + "phi4": (_PHI4_LAYER_KEYS, False), } # Converters whose per-layer map stands alone instead of extending _LAYER_KEYS. -_STANDALONE = {"lfm2", "lfm3"} +_STANDALONE = {"lfm2", "lfm3", "phi4"} # A conv layer and an attention layer carry different tensors, so the per-layer count that # the other families check cannot apply; LFM2 is checked by total instead. _MIXED_LAYERS = {"lfm2", "lfm3"} @@ -128,6 +148,23 @@ def unpermute(weight, n_heads: int): ) +def _phi4_parts(fused: str, tensor, config: dict) -> tuple: + """A fused Phi-4-mini projection cut where HF cuts it, refusing any other shape.""" + rows, cols = tensor.shape + if fused == "feed_forward.w13.weight": + hidden = int(config["intermediate_size"]) + if rows != 2 * hidden: + raise ValueError(f"gate_up_proj has {rows} rows, expected 2 x intermediate_size {hidden}") + return tensor[:hidden], tensor[hidden:] + n_heads = int(config["num_attention_heads"]) + head_dim = int(config.get("head_dim") or int(config["hidden_size"]) // n_heads) + q = n_heads * head_dim + kv = int(config.get("num_key_value_heads") or n_heads) * head_dim + if rows != q + 2 * kv: + raise ValueError(f"qkv_proj has {rows} rows, expected {q} + 2 x {kv}") + return tensor[:q], tensor[q : q + kv], tensor[q + kv :] + + def _shards(model_dir: Path) -> list[Path]: index = model_dir / "model.safetensors.index.json" if index.exists(): @@ -183,6 +220,12 @@ def convert(model_dir: Path, output: Path, converter: str, config: dict) -> dict for name, part in zip(split[1], torch.chunk(tensor, 3, dim=0), strict=True): state[head + name] = part continue + fused = next((f for f in _PHI4_FUSED if target.endswith(f)), None) if converter == "phi4" else None + if fused: + head = target[: -len(fused)] + for name, part in zip(_PHI4_FUSED[fused], _phi4_parts(fused, tensor, config), strict=True): + state[head + name] = part + continue if permute and target.endswith("attention.wq.weight"): tensor = unpermute(tensor, n_heads) elif permute and target.endswith("attention.wk.weight"): @@ -210,7 +253,9 @@ def convert(model_dir: Path, output: Path, converter: str, config: dict) -> dict if not widths or min(widths) < 6: raise ValueError(f"a layer is missing tensors: widths {widths}") else: - expected_per_layer = len(_LAYER_KEYS) + len(CONVERTERS[converter][0]) + # Phi-4-mini's six HF tensors per layer become Llama's nine. + extra = 0 if converter == "phi4" else len(CONVERTERS[converter][0]) + expected_per_layer = len(_LAYER_KEYS) + extra if per_layer != n_layers * expected_per_layer: raise ValueError(f"expected {n_layers} layers x {expected_per_layer} tensors, found {per_layer}") diff --git a/pipeline/eligibility.py b/pipeline/eligibility.py index 487ab3c..e4704a8 100644 --- a/pipeline/eligibility.py +++ b/pipeline/eligibility.py @@ -22,6 +22,7 @@ "lfm2": {"lfm25"}, # Ahead of the app (naming.AHEAD_OF_APP): it has no LFM3 template yet. "lfm3": {"lfm3"}, + "phi4_mini": {"phi4mini"}, } # What a release of each family is called. The architecture class alone is not enough: a # later generation can keep the class (a text-only "Qwen3.8" on Qwen3ForCausalLM would pass @@ -34,6 +35,9 @@ "llama": re.compile(r"llama-?3\.2(?![.\d])|smollm2(?!\d)", re.IGNORECASE), "gemma3": re.compile(r"gemma-?3(?![.\dn])", re.IGNORECASE), "smollm3": re.compile(r"smollm3(?!\d)", re.IGNORECASE), + # Phi-4-mini only: Phi-3 and Phi-3.5 share the architecture class, and the app's template + # is Phi-4-mini's (Phi4Prompt). + "phi4_mini": re.compile(r"phi-?4-mini(?![-.]?\d)", re.IGNORECASE), } _INSTRUCT_TOKENS = {"instruct", "it", "chat"} # Families whose chat models ship with no suffix at all and whose base models carry "-Base". diff --git a/pipeline/families.py b/pipeline/families.py index 7a90f89..917f827 100644 --- a/pipeline/families.py +++ b/pipeline/families.py @@ -71,6 +71,13 @@ def supports(self, backend: str) -> bool: _LFM3_NO_VULKAN = "the Vulkan delegate has no kernel for the short convolution (as for LFM2)" _LFM3_NO_QNN = "not in ExecuTorch 1.5.1's Qualcomm SUPPORTED_LLM_MODELS registry" _LFM3_NO_MTK = "no LFM3 definition in ExecuTorch 1.5.1's examples/mediatek" +# Phi-4-mini has upstream paths on both NPUs (Qualcomm's registry entry phi_4_mini, MediaTek's +# phi.py), neither of which this pipeline has wired or measured yet. +_PHI4_NO_QNN = "ExecuTorch 1.5.1's Qualcomm registry has phi_4_mini, but execupack has not wired or validated it" +_PHI4_NO_MTK = ( + "ExecuTorch 1.5.1's examples/mediatek has phi.py (model_type phi4, where HF says phi3), " + "but execupack has not validated its LongRoPE" +) FAMILIES: tuple[Family, ...] = ( Family("qwen3", ("Qwen3ForCausalLM",)), @@ -96,6 +103,10 @@ def supports(self, backend: str) -> bool: ("Gemma3ForCausalLM",), {"xnnpack": _NOT_IN_EXPORT_LLM, "vulkan": _NOT_IN_EXPORT_LLM, "mtk": _MTK_GEMMA3}, ), + # Phi-4-mini: Llama's block with fused q/k/v and gate/up projections (pipeline/convert.py + # splits them), partial RoPE on 96 of 128 head dims, and LongRoPE. ExecuTorch 1.5.1 builds + # it from the same transformer (examples/models/phi_4_mini). + Family("phi4_mini", ("Phi3ForCausalLM",), {"qnn": _PHI4_NO_QNN, "mtk": _PHI4_NO_MTK}), Family( "smollm3", ("SmolLM3ForCausalLM",), @@ -438,9 +449,67 @@ def xnnpack_plan(family: Family, config: dict) -> XnnpackPlan: params.update(use_scaled_rope=True, use_hf_rope=False, attention_qkv_bias=False) return XnnpackPlan("llama3_2", params, "llama") + if family.key == "phi4_mini": + return _phi4_mini_plan(c, params) + raise UnsupportedModel(f"no XNNPACK recipe for family {family.key}") +def _phi4_mini_plan(c: dict, params: dict) -> XnnpackPlan: + """Phi-4-mini's params, as ExecuTorch 1.5.1's examples/models/phi_4_mini/config/config.json + spells them, from the checkpoint's own config. + + LongRoPE (rope.hf_precompute_freqs_cis) divides the inverse frequencies by + ``short_factor`` or ``long_factor`` and scales cos and sin by ``attention_factor``, which + it derives from max over original position embeddings as HF does (1.19 here). HF picks + the factor per forward: short while the sequence is at most + ``original_max_position_embeddings`` (4096), long beyond. A precomputed table cannot, so + ExecuTorch picks once, by the table's length: long when it is over 4096. + + That length is the window -- unless ``ffn_dim_multiplier`` is set, in which case Rope + builds the table at twice the window (rope.py, "Sharded checkpoint"), and upstream's own + config sets it to 1. A 4k export would then run every prompt on the long factors, which + HF only uses past 4096 tokens. The multiplier does nothing else here (model_args reads it + only when hidden_dim is unset), so it is dropped: 2k and 4k windows use HF's short + factors exactly, and only windows past 4096 -- whose positions the short factors were + never trained for -- take the long ones, short prompts included. The gate grades that + against fp32 on short prompts, where HF uses the short factors. + """ + _require(c.get("model_type") == "phi3", f"model_type {c.get('model_type')!r} is not phi3") + _require(not c.get("attention_bias"), "Phi-4-mini with attention biases is not supported") + _require(not c.get("lm_head_bias"), "Phi-4-mini with an lm_head bias is not supported") + _require(bool(c.get("tie_word_embeddings")), "Phi-4-mini with untied embeddings is not validated") + partial = float(c.get("partial_rotary_factor") or 1.0) + _require(partial < 1.0, "full-width RoPE is Phi-3's, not Phi-4-mini's, and is not validated") + scaling = rope_scaling(c) + kind = (scaling or {}).get("rope_type") or (scaling or {}).get("type") + _require(kind == "longrope", f"RoPE scaling {kind!r}; the Phi-4-mini recipe implements longrope") + short, long = scaling.get("short_factor"), scaling.get("long_factor") + rotary_pairs = int(params["head_dim"] * partial) // 2 + _require( + isinstance(short, list) and isinstance(long, list) and len(short) == len(long) == rotary_pairs, + f"LongRoPE factors must be {rotary_pairs} long each", + ) + original = c.get("original_max_position_embeddings") or scaling.get("original_max_position_embeddings") + _require(isinstance(original, int) and original > 0, "LongRoPE without original_max_position_embeddings") + maximum = int(c["max_position_embeddings"]) + _require(maximum > original, "max_position_embeddings is not above the original context") + params.pop("ffn_dim_multiplier") + params.update( + use_scaled_rope=False, + use_hf_rope=True, + attention_qkv_bias=False, + partial_rotary_factor=partial, + original_max_position_embeddings=original, + max_position_embeddings=maximum, + rope_scaling_short_factor=[float(f) for f in short], + rope_scaling_long_factor=[float(f) for f in long], + ) + if scaling.get("attention_factor") is not None: + params["rope_scaling_attention_factor"] = float(scaling["attention_factor"]) + return XnnpackPlan("phi_4_mini", params, "phi4") + + def _lfm3_plan(c: dict) -> XnnpackPlan: """LFM3's params for pipeline/lfm3_model.py, from the fields of ExecuTorch's ModelArgs. diff --git a/pipeline/gate.py b/pipeline/gate.py index 87675a6..98602e6 100644 --- a/pipeline/gate.py +++ b/pipeline/gate.py @@ -94,17 +94,31 @@ def prompts(tokenizer) -> dict[str, str]: near tie that a 4-bit file broke the other way on two quiet rows without calling anything (2026-10-31). With the block closed in the prompt, the next token is the answer or the tool call, for every family; templates without the switch render as before. + + A reasoning-only template has no switch to read: Qwen3-4B-Thinking-2507's ends every + generation prompt in an open ```` whatever ``enable_thinking`` says, which would + put the first word of the reasoning back in the graded position. That block is closed + here the way Qwen3's own template closes it when the switch is off, and the way the app + sends a Qwen3 turn with thinking off (Qwen3Prompt.render, ``\\n\\n\\n\\n``). """ from pipeline import calibration def app(turn: str) -> str: - return calibration.render_app(tokenizer, turn, thinking=False) + return close_open_think(calibration.render_app(tokenizer, turn, thinking=False)) out = {q: app(f"{q}\n\n{calibration.TRAILER.format(subject=s)}") for q, s in SEARCH} out.update({q: app(q) for q in QUIET}) return out +def close_open_think(text: str) -> str: + """``text`` with a think block it leaves open at its very end closed empty.""" + head = text.rstrip("\n") + if head.endswith(""): + return head + "\n\n\n\n" + return text + + def reference(model, tokenizer) -> dict: """What the unquantised model would do: its top token per prompt, and how sure it is.""" import torch diff --git a/pipeline/naming.py b/pipeline/naming.py index e2ab9ae..9357533 100644 --- a/pipeline/naming.py +++ b/pipeline/naming.py @@ -128,8 +128,16 @@ def nominal_billions(name: str) -> float | None: return number / 1000 if hint.endswith("M") else number +# Releases whose own name carries no size. The app reads the size from the name +# (HubModel.parameterHint) and this pipeline refuses a name without one, so their outputs are +# named with it, appended: microsoft/Phi-4-mini-instruct is 3,836,021,760 parameters by the +# Hub's safetensors count and "3.8B" on its own model card. +SIZED_NAMES = {"microsoft/Phi-4-mini-instruct": "Phi-4-mini-instruct-3.8B"} + + def source_name(model_id: str) -> str: - return model_id.rsplit("/", 1)[-1] + """The name outputs are built from: the repo's own, with the size added where it has none.""" + return SIZED_NAMES.get(model_id) or model_id.rsplit("/", 1)[-1] def output_repo(model_id: str, hub_org: str, suffix: str) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index 2a840e7..66691a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,7 @@ "HuggingFaceTB/SmolLM2-360M-Instruct": 361_821_120, "Qwen/Qwen3-30B-A3B": 30_532_122_624, "google/gemma-3-1b-it": 999_885_952, + "microsoft/Phi-4-mini-instruct": 3_836_021_760, } CONFIG_FILES = { "Qwen/Qwen3-0.6B": "qwen3-0.6b", @@ -33,6 +34,7 @@ "HuggingFaceTB/SmolLM2-360M-Instruct": "smollm2-360m-instruct", "Qwen/Qwen3-30B-A3B": "qwen3-30b-a3b", "google/gemma-3-1b-it": "gemma-3-1b-it", + "microsoft/Phi-4-mini-instruct": "phi-4-mini-instruct", } diff --git a/tests/fixtures/et-phi_4_mini.params.json b/tests/fixtures/et-phi_4_mini.params.json new file mode 100644 index 0000000..59f4440 --- /dev/null +++ b/tests/fixtures/et-phi_4_mini.params.json @@ -0,0 +1,19 @@ +{ + "dim": 3072, + "ffn_dim_multiplier": 1, + "hidden_dim": 8192, + "n_heads": 24, + "n_kv_heads": 8, + "n_layers": 32, + "norm_eps": 1e-05, + "rope_theta": 10000.0, + "use_scaled_rope": false, + "vocab_size": 200064, + "use_hf_rope": true, + "partial_rotary_factor": 0.75, + "attention_qkv_bias": false, + "original_max_position_embeddings": 4096, + "max_position_embeddings": 131072, + "rope_scaling_short_factor": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "rope_scaling_long_factor": [1.0, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77] +} diff --git a/tests/fixtures/phi-4-mini-instruct.config.json b/tests/fixtures/phi-4-mini-instruct.config.json new file mode 100644 index 0000000..679e70d --- /dev/null +++ b/tests/fixtures/phi-4-mini-instruct.config.json @@ -0,0 +1,144 @@ +{ + "_name_or_path": "Phi-4-mini-instruct", + "architectures": [ + "Phi3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_phi3.Phi3Config", + "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM", + "AutoTokenizer": "Xenova/gpt-4o" + }, + "bos_token_id": 199999, + "embd_pdrop": 0.0, + "eos_token_id": 199999, + "full_attn_mod": 1, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "interpolate_factor": 1, + "lm_head_bias": false, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "phi3", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "original_max_position_embeddings": 4096, + "pad_token_id": 199999, + "partial_rotary_factor": 0.75, + "resid_pdrop": 0.0, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "long_factor": [ + 1, + 1.118320672, + 1.250641126, + 1.398617824, + 1.564103225, + 1.74916897, + 1.956131817, + 2.187582649, + 2.446418898, + 2.735880826, + 3.059592084, + 3.421605075, + 3.826451687, + 4.279200023, + 4.785517845, + 5.351743533, + 5.984965424, + 6.693110555, + 7.485043894, + 8.370679318, + 9.36110372, + 10.4687158, + 11.70738129, + 13.09260651, + 14.64173252, + 16.37415215, + 18.31155283, + 20.47818807, + 22.90118105, + 25.61086418, + 28.64115884, + 32.03, + 32.1, + 32.13, + 32.23, + 32.6, + 32.61, + 32.64, + 32.66, + 32.7, + 32.71, + 32.93, + 32.97, + 33.28, + 33.49, + 33.5, + 44.16, + 47.77 + ], + "short_factor": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "type": "longrope" + }, + "rope_theta": 10000.0, + "sliding_window": 262144, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.45.0", + "use_cache": true, + "vocab_size": 200064 +} diff --git a/tests/test_convert.py b/tests/test_convert.py index 4f5cfd9..eb0e28e 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -157,3 +157,42 @@ def test_the_fused_convolution_projection_splits_into_three(): target, parts = convert.SPLIT_THREE["lfm2"] assert target == "conv.in_proj.weight" assert parts == ("conv.B_proj.weight", "conv.C_proj.weight", "conv.x_proj.weight") + + +def test_phi4_fused_projections_split_where_hf_splits_them(tmp_path): + config = {**tiny_config(), "hidden_size": HIDDEN, "intermediate_size": FFN, "head_dim": HEAD_DIM} + plain = tiny_checkpoint() + fused = {k: v for k, v in plain.items() if "_proj" not in k or "o_proj" in k or "down_proj" in k} + for i in range(2): + p = f"model.layers.{i}." + fused[p + "self_attn.qkv_proj.weight"] = torch.cat( + [plain[p + f"self_attn.{n}_proj.weight"] for n in "qkv"], dim=0 + ) + fused[p + "mlp.gate_up_proj.weight"] = torch.cat( + [plain[p + "mlp.gate_proj.weight"], plain[p + "mlp.up_proj.weight"]], dim=0 + ) + phi_dir = tmp_path / "phi" + phi_dir.mkdir() + safetensors_torch.save_file(fused, str(phi_dir / "model.safetensors")) + convert.convert(phi_dir, tmp_path / "phi.pth", "phi4", config) + got = torch.load(tmp_path / "phi.pth") + for i in range(2): + p, e = f"model.layers.{i}.", f"layers.{i}." + assert torch.equal(got[e + "attention.wq.weight"], plain[p + "self_attn.q_proj.weight"]) + assert torch.equal(got[e + "attention.wk.weight"], plain[p + "self_attn.k_proj.weight"]) + assert torch.equal(got[e + "attention.wv.weight"], plain[p + "self_attn.v_proj.weight"]) + assert torch.equal(got[e + "feed_forward.w1.weight"], plain[p + "mlp.gate_proj.weight"]) + assert torch.equal(got[e + "feed_forward.w3.weight"], plain[p + "mlp.up_proj.weight"]) + assert not any(k.endswith(("wqkv.weight", "w13.weight")) for k in got) + assert torch.equal(got["output.weight"], got["tok_embeddings.weight"]) + + +def test_phi4_fused_projection_of_the_wrong_height_is_refused(tmp_path): + config = {**tiny_config(layers=1), "hidden_size": HIDDEN, "intermediate_size": FFN + 2, "head_dim": HEAD_DIM} + sd = {k: v for k, v in tiny_checkpoint(layers=1).items() if "_proj" not in k or "o_proj" in k or "down_proj" in k} + sd["model.layers.0.self_attn.qkv_proj.weight"] = torch.zeros((HEADS + 2 * KV_HEADS) * HEAD_DIM, HIDDEN) + sd["model.layers.0.mlp.gate_up_proj.weight"] = torch.zeros(2 * FFN, HIDDEN) + (tmp_path / "m").mkdir() + safetensors_torch.save_file(sd, str(tmp_path / "m" / "model.safetensors")) + with pytest.raises(ValueError, match="gate_up_proj has"): + convert.convert(tmp_path / "m", tmp_path / "out.pth", "phi4", config) diff --git a/tests/test_eligibility.py b/tests/test_eligibility.py index ad77085..b5e31cb 100644 --- a/tests/test_eligibility.py +++ b/tests/test_eligibility.py @@ -156,3 +156,16 @@ def test_a_chat_model_named_without_a_suffix_is_still_published(): # The other half of the rule: Qwen3 and LFM2.5-2.6B ship chat models with no suffix, and # refusing "base" by name alone would drop them. assert evaluate("Qwen/Qwen3-1.7B").eligible + + +def test_phi4_mini_is_eligible_for_cpu_and_gpu_only(source): + verdict = eligibility.evaluate(source("microsoft/Phi-4-mini-instruct"), settings.load()) + assert verdict.reasons == [] + assert (verdict.family, verdict.variant) == ("phi4_mini", "instruct") + assert verdict.export_backends == ["xnnpack", "vulkan"] + + +def test_phi_3_5_mini_shares_the_class_and_is_still_refused(source): + config = {**hf_config("microsoft/Phi-4-mini-instruct"), "partial_rotary_factor": 1.0} + verdict = eligibility.evaluate(source("microsoft/Phi-3.5-mini-instruct", config=config), settings.load()) + assert verdict.export_backends == [] diff --git a/tests/test_families.py b/tests/test_families.py index eec040a..93c4a87 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -217,3 +217,55 @@ def test_one_attention_layer_per_chunk_is_only_for_hybrids(): # An explicit all-attention layout is still not a hybrid: no chunk per layer for it. qwen["layer_types"] = ["full_attention"] * int(qwen["num_hidden_layers"]) assert families.mtk_chunks_one_attention_each(qwen) is None + + +PHI4 = "microsoft/Phi-4-mini-instruct" + + +def test_phi4_mini_params_equal_executorchs_own_but_the_table_doubling_multiplier(): + # ExecuTorch's config sets ffn_dim_multiplier 1, which makes Rope build its table at twice + # the window and so run a 4k export on LongRoPE's long factors (families._phi4_mini_plan). + expected = expected_params("et-phi_4_mini.params.json") + expected.pop("ffn_dim_multiplier") + plan = plan_for(hf_config(PHI4)) + assert plan.params == expected + assert (plan.model_class, plan.converter) == ("phi_4_mini", "phi4") + + +def test_phi4_mini_shapes_outside_the_validated_one_are_refused(): + for change, match in ( + ({"partial_rotary_factor": 1.0}, "full-width RoPE"), + ({"tie_word_embeddings": False}, "untied"), + ({"rope_scaling": None}, "longrope"), + ({"attention_bias": True}, "attention biases"), + ): + config = copy.deepcopy(hf_config(PHI4)) + config.update(change) + with pytest.raises(families.UnsupportedModel, match=match): + plan_for(config) + config = copy.deepcopy(hf_config(PHI4)) + config["rope_scaling"]["long_factor"] = config["rope_scaling"]["long_factor"][:-1] + with pytest.raises(families.UnsupportedModel, match="48 long"): + plan_for(config) + + +@pytest.mark.parametrize("window, factor", [(2048, "short"), (4096, "short"), (8192, "long"), (32768, "long")]) +def test_phi4_mini_rope_table_takes_hf_short_factors_up_to_4096(window, factor): + rope = pytest.importorskip("executorch.examples.models.llama.rope") + model_args = pytest.importorskip("executorch.examples.models.llama.model_args") + torch = pytest.importorskip("torch") + params = plan_for(hf_config(PHI4)).params + built = rope.Rope(model_args.ModelArgs(**params, max_context_len=window, max_seq_len=min(window, 2048))) + assert built.freqs_cos.shape[0] == window + chosen = params[f"rope_scaling_{factor}_factor"] + want_cos, _ = rope.hf_precompute_freqs_cis( + params["head_dim"], + window, + params["rope_theta"], + params["partial_rotary_factor"], + short_factor=chosen, + long_factor=chosen, + original_max_pos=4096, + max_position_embeddings=131072, + ) + assert torch.equal(built.freqs_cos, want_cos) diff --git a/tests/test_gate.py b/tests/test_gate.py index 25dd268..f863f28 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -137,3 +137,64 @@ def test_measuring_gate_records_throughput_and_passes(): assert result["decode_tok_per_sec"] == 42.12 assert result["prompt_tokens"] == 16 assert result["generated_tokens"] == 32 + + +class _Template: + """A chat template reduced to its generation prompt, as three real families end it.""" + + def __init__(self, opener): + self.opener = opener + + def apply_chat_template(self, messages, tools=None, tokenize=False, add_generation_prompt=True, **extra): + body = "".join(f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n" for m in messages) + return body + "<|im_start|>assistant\n" + self.opener(extra.get("enable_thinking", True)) + + +def test_a_reasoning_only_template_has_its_think_block_closed_like_the_switch_closes_qwen3s(): + # Qwen3-4B-Thinking-2507 opens the block whatever enable_thinking says; hybrid Qwen3 + # closes it empty when the switch is off. The gate has to grade both on the decision. + thinking_only = gate.prompts(_Template(lambda thinking: "\n")) + hybrid = gate.prompts(_Template(lambda thinking: "" if thinking else "\n\n\n\n")) + assert thinking_only == hybrid + assert all(p.endswith("<|im_start|>assistant\n\n\n\n\n") for p in thinking_only.values()) + + +def test_a_template_without_a_think_block_is_graded_as_it_renders(): + plain = gate.prompts(_Template(lambda thinking: "")) + assert all(p.endswith("<|im_start|>assistant\n") for p in plain.values()) + assert gate.close_open_think("a \nreasoning\n\nanswer") == "a \nreasoning\n\nanswer" + + +# microsoft/Phi-4-mini-instruct's chat_template (tokenizer_config.json, rev cfbefacb), verbatim. +PHI4_TEMPLATE = ( + "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and " + "message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + " + "message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + " + "message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}" + "{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}" +) + + +class _Jinja: + """apply_chat_template reduced to rendering the template, as transformers does.""" + + def __init__(self, template): + import jinja2 + + self.chat_template = template + self._template = jinja2.Environment().from_string(template) + + def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True, **extra): + return self._template.render( + messages=messages, add_generation_prompt=add_generation_prompt, eos_token="", **extra + ) + + +def test_phi4_mini_gets_the_tool_schema_on_its_system_turn_as_the_app_sends_it(): + import json + + rendered = gate.prompts(_Jinja(PHI4_TEMPLATE))["What is Concord the capital of?"] + # Phi4Prompt.render: the system text, then the tools as one JSON array between markers. + head = f"<|system|>{calibration.APP_SYSTEM}<|tool|>{json.dumps(calibration.APP_TOOLS)}<|/tool|><|end|>" + assert rendered.startswith(head) + assert rendered.endswith("<|assistant|>") diff --git a/tests/test_naming.py b/tests/test_naming.py index db7d88d..176f0e2 100644 --- a/tests/test_naming.py +++ b/tests/test_naming.py @@ -105,3 +105,13 @@ def test_window_label(): assert naming.window_label(32768) == "32k" assert naming.window_label(8192) == "8k" assert naming.window_label(3000) == "3000" + + +def test_phi4_mini_is_named_with_its_size_and_reads_as_phi4mini_in_the_app(): + repo = naming.output_repo("microsoft/Phi-4-mini-instruct", "experimentalmachines", "-ExecuTorch") + assert repo == "experimentalmachines/Phi-4-mini-instruct-3.8B-ExecuTorch" + for backend in ("xnnpack", "vulkan"): + path = f"{backend}/" + naming.cpu_gpu_file("microsoft/Phi-4-mini-instruct", backend, "8da4w-gptq", 4096) + assert naming.check_app_rules(repo, path, backend) == [] + assert naming.app_family(naming.app_model_name(repo, path)) == "phi4mini" + assert naming.app_size_hint(repo.split("/")[1]) == "3.8B" From 806e881d8160f826e93d628f7a33c992b657e029 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 10:07:08 +0800 Subject: [PATCH 02/10] verify and publish: a gated report has to have passed its gate verify skipped any report that already had a smoke result, even when it was asked to gate. So a first run that passed the smoke test and then failed, or never reached, the decision gate left a report that a second run returned as verified with nothing gated. publish-hf checked only the smoke result, so that report could be published. Now verify gates it again, and publish refuses a report whose gate failed. A report with no gate at all (Vulkan, or a run without an fp32 reference) publishes on its smoke test, as before. Found by codex review. Co-Authored-By: Claude Opus 5.5 --- pipeline/publish.py | 7 ++++++ pipeline/verify.py | 7 +++++- tests/test_publish.py | 12 ++++++++++ tests/test_verify.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 tests/test_verify.py diff --git a/pipeline/publish.py b/pipeline/publish.py index 25c4642..9ba0736 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -90,6 +90,13 @@ def publish_hf( ) if not smoke.get("passed"): raise ValueError(f"{report['files'][0]['path']} failed its smoke test: {'; '.join(smoke.get('problems', []))}") + # A report that was gated must have passed it. One with no gate at all (Vulkan, or a run + # without GPTQ and so without an fp32 reference) is published on its smoke test, as before. + verdict = report.get("gate") + if verdict is not None and not verdict.get("passed"): + raise ValueError( + f"{report['files'][0]['path']} failed its decision gate: {'; '.join(verdict.get('problems', []))}" + ) repo_id = report["output_repo"] folder = _folder(backend, target) label = naming.window_label(report["window"]["context"]) diff --git a/pipeline/verify.py b/pipeline/verify.py index 02c0467..4d1ab94 100644 --- a/pipeline/verify.py +++ b/pipeline/verify.py @@ -36,7 +36,12 @@ def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None gates: list[dict] = [] for path in reports(out_dir, backend): report = json.loads(path.read_text()) - if report.get("smoke") is not None: + # Verified already, unless a gate is asked for that this report has not passed: a + # first run that passed the smoke test and then failed (or never reached) the gate + # leaves a smoke result behind, and skipping on that alone let a second run report + # success with nothing gated (codex review, 2026-11-09). + gated = gate_reference is None or (report.get("gate") or {}).get("passed") + if report.get("smoke") is not None and gated: print(f"==> {path.name}: already verified, leaving it") results.append(report["smoke"]) continue diff --git a/tests/test_publish.py b/tests/test_publish.py index 9519e76..2c35a34 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -246,3 +246,15 @@ def test_load_report_wants_exactly_one_report(tmp_path): with pytest.raises(ValueError, match="pass --context"): publish.load_report(tmp_path, "xnnpack") assert publish.load_report(tmp_path, "xnnpack", context=8192)["window"]["context"] == 8192 + + +def test_a_report_that_failed_its_decision_gate_is_never_published(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"pte") + failed = {**report(2048), "gate": {"passed": False, "problems": ["kept fp32's choice on 7 of 8 calling rows"]}} + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(failed)) + monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) + with pytest.raises(ValueError, match="failed its decision gate: kept fp32's choice on 7 of 8"): + publish.publish_hf(out, "xnnpack") diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000..8c04f60 --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,52 @@ +"""verify decides what still needs checking from the report it finds.""" + +import json + +import pytest + +from pipeline import verify + + +class Ran(Exception): + """The smoke test was reached: verify did not skip the report.""" + + +def write(tmp_path, report): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "xnnpack" / "x-2k.pte").write_bytes(b"pte") + base = { + "window": {"context": 2048}, + "files": [{"path": "xnnpack/x-2k.pte"}], + "tokenizer": "tokenizer.json", + "source": {"id": "org/x", "sha": "0" * 40, "variant": "instruct"}, + } + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps({**base, **report})) + return out + + +@pytest.fixture +def reaches_smoke(monkeypatch): + def fetch(*args, **kwargs): + raise Ran + + monkeypatch.setattr(verify.hub, "fetch", fetch) + + +@pytest.mark.parametrize("gate", [None, {"passed": False, "problems": ["7 of 8"]}]) +def test_a_smoke_tested_report_is_gated_again_when_its_gate_is_missing_or_failed(tmp_path, reaches_smoke, gate): + report = {"smoke": {"passed": True}} | ({"gate": gate} if gate else {}) + out = write(tmp_path, report) + reference = tmp_path / "gate.json" + reference.write_text("{}") + with pytest.raises(Ran): + verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference) + + +def test_a_report_that_passed_both_is_left_alone(tmp_path, reaches_smoke): + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True}}) + reference = tmp_path / "gate.json" + reference.write_text("{}") + assert verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference)["passed"] + out_ungated = write(tmp_path / "b", {"smoke": {"passed": True}}) + assert verify.run(out_ungated, "xnnpack", tmp_path / "work")["passed"] From fbc2917f25928367aa1e1e23a093d71ae12beff4 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 10:37:17 +0800 Subject: [PATCH 03/10] Phi-4-mini: an int8 output head, and GPTQ codes for the Vulkan export Phi-4-mini's GPTQ CPU export missed the decision gate by one row: it kept 7 of 8 tool calls, at a mean of 0.45 against fp32's 0.66. Measured eager on the gate's prompts against fp32, holding everything else fixed, the int4 output head was the cost: - int4 everywhere kept 7/8 at 0.457 (the .pte measured 0.452); - an fp32 head kept 8/8 at 0.521; - an int8 per-channel head kept 8/8 at 0.526. The head is the tied 200,064 x 3072 embedding, a sixth of the model, and fp32 is itself only 0.40 to 0.78 sure on these rows. XnnpackPlan.output_bits sets the head's bits; Phi-4-mini uses 8, which adds about 0.27 GB to the file. The head gets int8 weights per channel with the same dynamic int8 activations, and its int4 codes are skipped. export-vulkan takes --codes. The Vulkan delegate runs the same torchao linears, and round-to-nearest int4 kept none of Phi's 8 tool calls (0.05 against 0.66). There is no Vulkan runtime on the host, so a Vulkan file is gated through the XNNPACK file solved from the same codes. Co-Authored-By: Claude Opus 5.5 --- pipeline/__main__.py | 9 +++++ pipeline/export_with_codes.py | 75 +++++++++++++++++++++++++---------- pipeline/export_xnnpack.py | 30 +++++++++++--- pipeline/families.py | 11 ++++- tests/test_solve.py | 24 +++++++++++ 5 files changed, 122 insertions(+), 27 deletions(-) diff --git a/pipeline/__main__.py b/pipeline/__main__.py index daed58a..e732b4d 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -126,6 +126,7 @@ def _export_vulkan(args) -> int: Path(args.work), context=args.context, keep_work=args.keep_work, + codes=Path(args.codes) if args.codes else None, backend="vulkan", ) ) @@ -480,6 +481,14 @@ def main(argv: list[str] | None = None) -> int: vulkan.add_argument( "--keep-work", action="store_true", help="keep the work dir after success (a failure always leaves it)" ) + # The Vulkan delegate runs the same torchao Int8DynamicActivationIntxWeight linears as + # XNNPACK, so the solve's codes go in the same way. There is no Vulkan runtime on the + # host to gate the file, so its gate is the XNNPACK file solved from the same codes. + vulkan.add_argument( + "--codes", + default=None, + help="codes.pt from `solve`; without it the int4 weights are rounded to nearest", + ) vulkan.set_defaults(func=_export_vulkan) qnn = commands.add_parser("export-qnn", help="compile a Qualcomm HTP .pte for one chip") diff --git a/pipeline/export_with_codes.py b/pipeline/export_with_codes.py index e8cfa83..9c5827a 100644 --- a/pipeline/export_with_codes.py +++ b/pipeline/export_with_codes.py @@ -63,6 +63,52 @@ def inject(model, codes: dict, log=print) -> tuple[int, list[str]]: return injected, left +def quantizer(upstream, codes: dict, output_int8: bool): + """``upstream`` (source_transformation.quantize.quantize) with the solved codes put in, + and for ``output_int8`` the output head at int8 per channel rather than int4.""" + import torch + + if output_int8: + # The head is not int4 here, so its int4 codes have nowhere to go. + codes = {k: v for k, v in codes.items() if k != "output"} + + def quantize(model, qmode, *rest, **kwargs): + if qmode != "8da4w": + return upstream(model, qmode, *rest, **kwargs) + head = None + if output_int8: + # Hidden from the int4 pass, which takes every nn.Linear, then given int8 weights + # per channel with the same dynamic int8 activations + # (Int8DynamicActivationIntxWeightConfig, as upstream's own 8da8w mode builds them). + head, model.output = model.output, torch.nn.Identity() + quantised = upstream(model, qmode, *rest, **kwargs) + if head is not None: + from torchao.quantization import Int8DynamicActivationIntxWeightConfig, quantize_ + from torchao.quantization.granularity import PerAxis + + with torch.no_grad(): + quantize_( + head, + Int8DynamicActivationIntxWeightConfig(weight_dtype=torch.int8, weight_granularity=PerAxis(0)), + ) + quantised.output = head + print("output head quantized to int8 per channel") + if not codes: + return quantised + injected, _ = inject(quantised, codes) + # Every code must find its tensor. If the solve was run against a different + # checkpoint, or a name moved, the export would otherwise finish, be labelled GPTQ, + # and carry weights that were quietly rounded instead. + if injected != len(codes): + missing = sorted(set(codes) - {n for n, _ in quantised.named_modules()}) + raise ValueError( + f"{len(codes)} solved tensors but {injected} were injected; these codes match no module: {missing[:5]}" + ) + return quantised + + return quantize + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True, help="the export_llm YAML the exporter wrote") @@ -72,6 +118,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="clear the short convolution's state at position zero (pipeline/lfm2_state.py)", ) + parser.add_argument( + "--output-int8", + action="store_true", + help="quantize the output head to int8 per channel instead of int4 (XnnpackPlan.output_bits)", + ) parser.add_argument( "--lfm3", action="store_true", @@ -93,31 +144,13 @@ def main(argv: list[str] | None = None) -> int: lfm3_model.apply() print("applied the LFM3 model definition (pipeline/lfm3_model.py)") - if args.codes: + if args.codes or args.output_int8: from executorch.examples.models.llama.source_transformation import quantize as quantize_module # weights_only: the codes file is an artifact passed between CI jobs, and it holds # nothing but tensors, so there is no reason to let it unpickle arbitrary objects. - codes = torch.load(args.codes, map_location="cpu", weights_only=True) - upstream = quantize_module.quantize - - def quantize(model, qmode, *rest, **kwargs): - quantised = upstream(model, qmode, *rest, **kwargs) - if qmode != "8da4w": - return quantised - injected, _ = inject(quantised, codes) - # Every code must find its tensor. If the solve was run against a different - # checkpoint, or a name moved, the export would otherwise finish, be labelled - # GPTQ, and carry weights that were quietly rounded instead. - if injected != len(codes): - missing = sorted(set(codes) - {n for n, _ in quantised.named_modules()}) - raise ValueError( - f"{len(codes)} solved tensors but {injected} were injected; " - f"these codes match no module: {missing[:5]}" - ) - return quantised - - quantize_module.quantize = quantize + codes = torch.load(args.codes, map_location="cpu", weights_only=True) if args.codes else {} + quantize_module.quantize = quantizer(quantize_module.quantize, codes, args.output_int8) from executorch.extension.llm.export.export_llm import main as export_llm_main diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 286f2af..f7e87b0 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -139,7 +139,13 @@ def pick_runner( def recipe_fields( - recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int, executorch: str | None = None + recipe, + qmode_label: str, + solved: bool, + backend: str, + prefill_chunk: int, + executorch: str | None = None, + output_bits: int = 4, ) -> dict: """The report's description of how the weights are stored, true to the recipe built. @@ -169,18 +175,22 @@ def recipe_fields( f"{prefill_chunk}, fp32 KV cache." ), } + head = output_bits == 8 return { **common, "group_size": recipe.group_size, "int4_codes": "gptq" if solved else "round-to-nearest", + **({"output_head": "int8 per channel"} if head else {}), "label": ( f"{qmode_label}-g{recipe.group_size}, int8 embeddings" + (", int4 codes solved with GPTQ" if solved else ", int4 codes rounded to nearest") + + (", int8 output head" if head else "") ), "description": ( f"ExecuTorch {version} `export_llm`: 8-bit dynamic activations " - f"and 4-bit weights in groups of {recipe.group_size}, int8 per-channel embeddings, " - f"{delegate}, prefill chunk {prefill_chunk}, fp32 KV cache." + f"and 4-bit weights in groups of {recipe.group_size}" + + (" (the output head 8-bit per channel)" if head else "") + + f", int8 per-channel embeddings, {delegate}, prefill chunk {prefill_chunk}, fp32 KV cache." ), } @@ -290,7 +300,8 @@ def run( # LFM3's model is this repository's (pipeline/lfm3_model.py), so its export always goes # through the wrapper too; its convolution clears its own state at position zero. state_fix = plan.converter == "lfm2" - if codes is not None or state_fix or plan.lfm3: + output_int8 = plan.output_bits == 8 and recipe.qmode == "8da4w" + if codes is not None or state_fix or plan.lfm3 or output_int8: command = [sys.executable, "-m", "pipeline.export_with_codes", "--config", str(config_path)] if codes is not None: command += ["--codes", str(codes)] @@ -298,6 +309,8 @@ def run( command += ["--lfm2-state-fix"] if plan.lfm3: command += ["--lfm3"] + if output_int8: + command += ["--output-int8"] print( f"==> export_llm (codes: {'gptq' if codes else 'round to nearest'}, state fix: {state_fix}, " f"lfm3: {plan.lfm3})" @@ -364,7 +377,14 @@ def run( "params": plan.params, "export_llm_qmode": QMODES[recipe.qmode], "embedding_hqq": recipe.embedding_hqq, - **recipe_fields(recipe, qmode_label, codes is not None, backend, min(cfg.prefill_chunk, window)), + **recipe_fields( + recipe, + qmode_label, + codes is not None, + backend, + min(cfg.prefill_chunk, window), + output_bits=plan.output_bits if recipe.qmode == "8da4w" else 4, + ), }, "window": { "context": window, diff --git a/pipeline/families.py b/pipeline/families.py index 917f827..cd9b666 100644 --- a/pipeline/families.py +++ b/pipeline/families.py @@ -367,6 +367,9 @@ class XnnpackPlan: model_class: str params: dict converter: str # key into pipeline.convert.CONVERTERS + # Bits for the output head's weights in an 8da4w export: 4 like every other linear, + # or 8 (per channel) where the int4 head is what costs the model its decisions. + output_bits: int = 4 @property def lfm3(self) -> bool: @@ -507,7 +510,13 @@ def _phi4_mini_plan(c: dict, params: dict) -> XnnpackPlan: ) if scaling.get("attention_factor") is not None: params["rope_scaling_attention_factor"] = float(scaling["attention_factor"]) - return XnnpackPlan("phi_4_mini", params, "phi4") + # The output head stays at 8 bits. Measured 2026-11-09 on the gate's prompts, eager, against + # fp32 (scratchpad expt.py): every linear int4 with GPTQ codes kept 7 of 8 tool calls at a + # mean of 0.457 against fp32's 0.663 -- the published recipe, and the .pte agreed (0.452) -- + # while the same with the head left in fp32 kept 8 of 8 at 0.521. Phi-4-mini's head is its + # tied 200,064 x 3072 embedding, a sixth of the model, and fp32 itself is unsure on these + # rows (0.40 to 0.78), so the head's rounding is enough to tip one. + return XnnpackPlan("phi_4_mini", params, "phi4", output_bits=8) def _lfm3_plan(c: dict) -> XnnpackPlan: diff --git a/tests/test_solve.py b/tests/test_solve.py index 1b35ae5..0fd5917 100644 --- a/tests/test_solve.py +++ b/tests/test_solve.py @@ -150,3 +150,27 @@ def named_modules(self): codes = {"linear": {"qdata": torch.zeros(4, 32, dtype=torch.int8), "scale": bad}} with pytest.raises(ValueError, match="not finite"): export_with_codes.inject(Module(), codes) + + +def test_an_int8_output_head_is_int8_per_channel_and_takes_no_int4_codes(): + pytest.importorskip("executorch.examples.models.llama.source_transformation.quantize") + from executorch.examples.models.llama.source_transformation.quantize import quantize as upstream + + from pipeline import export_with_codes + + class Tiny(torch.nn.Module): + def __init__(self): + super().__init__() + self.wq = torch.nn.Linear(64, 64, bias=False) + self.output = torch.nn.Linear(64, 128, bias=False) + + model = Tiny() + head = model.output.weight.detach().clone() + # The head's int4 codes are dropped rather than injected into an int8 tensor. + codes = {"output": {"qdata": torch.zeros(128, 64, dtype=torch.int8), "scale": torch.ones(128, 2)}} + out = export_with_codes.quantizer(upstream, codes, output_int8=True)(model, "8da4w", group_size=32) + w4, w8 = out.wq.weight, out.output.weight + assert int(w4.qdata.min()) >= -8 and int(w4.qdata.max()) <= 7 and tuple(w4.scale.shape) == (64, 2) + assert tuple(w8.scale.reshape(128, -1).shape) == (128, 1) # one scale per output channel + assert int(w8.qdata.max()) > 7 # the full int8 range, not the int4 grid + assert float((w8.dequantize() - head).abs().max()) < float(head.abs().max()) / 100 From c277c1d3e8bcebfb7f7ec91b3d53176c56fd6a60 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 10:43:13 +0800 Subject: [PATCH 04/10] Publish only what was gated, and tie a Vulkan GPTQ file to the gate that stands for it Codex review, round three: - publish-hf and publish-release share one check now. A GPTQ report must carry a passing gate of its own, because the solve always writes an fp32 reference, so a GPTQ report without a gate is one whose gate crashed or never ran. A Vulkan GPTQ report, which the host cannot run, must instead carry a passing gate proxy. publish-release checked nothing before. - export-vulkan --codes needs --gate-proxy: the XNNPACK report from the same source revision, window, codes (sha256, now recorded in every GPTQ report) and output head, whose own gate passed. It is checked before the export and recorded in the Vulkan report. - A codes file that leaves an int4 linear unsolved is refused, not exported as GPTQ with that linear rounded. - The size and phone-fit estimates price an int8 output head at a byte a weight. Co-Authored-By: Claude Opus 5.5 --- pipeline/__main__.py | 6 ++++ pipeline/export_with_codes.py | 11 ++++-- pipeline/export_xnnpack.py | 66 +++++++++++++++++++++++++++++++++-- pipeline/publish.py | 53 ++++++++++++++++++---------- pipeline/sizing.py | 16 ++++++--- tests/test_export_xnnpack.py | 46 ++++++++++++++++++++++++ tests/test_publish.py | 16 +++++++++ tests/test_solve.py | 29 +++++++++++++-- 8 files changed, 212 insertions(+), 31 deletions(-) diff --git a/pipeline/__main__.py b/pipeline/__main__.py index e732b4d..040ba74 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -127,6 +127,7 @@ def _export_vulkan(args) -> int: context=args.context, keep_work=args.keep_work, codes=Path(args.codes) if args.codes else None, + gate_proxy=Path(args.gate_proxy) if args.gate_proxy else None, backend="vulkan", ) ) @@ -489,6 +490,11 @@ def main(argv: list[str] | None = None) -> int: default=None, help="codes.pt from `solve`; without it the int4 weights are rounded to nearest", ) + vulkan.add_argument( + "--gate-proxy", + default=None, + help="with --codes: the XNNPACK export-report-.json from the same codes that passed its gate", + ) vulkan.set_defaults(func=_export_vulkan) qnn = commands.add_parser("export-qnn", help="compile a Qualcomm HTP .pte for one chip") diff --git a/pipeline/export_with_codes.py b/pipeline/export_with_codes.py index 9c5827a..98fae77 100644 --- a/pipeline/export_with_codes.py +++ b/pipeline/export_with_codes.py @@ -68,6 +68,7 @@ def quantizer(upstream, codes: dict, output_int8: bool): and for ``output_int8`` the output head at int8 per channel rather than int4.""" import torch + codes_given = bool(codes) if output_int8: # The head is not int4 here, so its int4 codes have nowhere to go. codes = {k: v for k, v in codes.items() if k != "output"} @@ -93,9 +94,15 @@ def quantize(model, qmode, *rest, **kwargs): ) quantised.output = head print("output head quantized to int8 per channel") - if not codes: + if not codes_given: return quantised - injected, _ = inject(quantised, codes) + injected, left = inject(quantised, codes) + # And every int4 tensor must have its code: a partial codes file would otherwise + # export, be labelled GPTQ, and carry the rest rounded (codex review, 2026-11-09). The + # int8 head has no int4 code by design. + rounded = [n for n in left if not (output_int8 and n == "output")] + if rounded: + raise ValueError(f"no solved codes for {len(rounded)} int4 linears: {rounded[:5]}") # Every code must find its tensor. If the solve was run against a different # checkpoint, or a name moved, the export would otherwise finish, be labelled GPTQ, # and carry weights that were quietly rounded instead. diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index f7e87b0..1557b15 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -138,6 +138,52 @@ def pick_runner( return None +def file_sha256(path: Path) -> str: + import hashlib + + digest = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 24), b""): + digest.update(block) + return digest.hexdigest() + + +def check_gate_proxy(path: Path | None, sha: str, window: int, codes_sha256: str, output_bits: int) -> dict: + """What stands in for a Vulkan file's gate: the XNNPACK report built from the same codes. + + The two delegates run the same torchao linears with the same codes, so the XNNPACK file + is the closest measurement of the Vulkan one the host can make. It counts only for the + same source revision, window (LongRoPE picks its factors by window), codes file and output + head, and only when its own gate passed; anything else would label an ungated file as + gated (codex review, 2026-11-09). + """ + if path is None: + raise ExportError("a Vulkan export with solved codes needs --gate-proxy, the XNNPACK report that gates it") + report = json.loads(Path(path).read_text()) + recipe, verdict = report.get("recipe") or {}, report.get("gate") or {} + problems = [] + if report.get("backend") != "xnnpack": + problems.append(f"it is a {report.get('backend')} report") + if (report.get("source") or {}).get("sha") != sha: + problems.append("it was built from another source revision") + if (report.get("window") or {}).get("context") != window: + problems.append(f"its window is {(report.get('window') or {}).get('context')}, not {window}") + if recipe.get("codes_sha256") != codes_sha256: + problems.append("it was built from other codes") + if (recipe.get("output_head") == "int8 per channel") != (output_bits == 8): + problems.append("its output head differs") + if not verdict.get("passed"): + problems.append("its decision gate did not pass" if verdict else "it has no decision gate") + if problems: + raise ExportError(f"gate proxy {path} does not stand for this export: {'; '.join(problems)}") + return { + "report": Path(path).name, + "file": report["files"][0]["path"], + "sha256": report["files"][0].get("sha256"), + "gate": {k: verdict.get(k) for k in ("passed", "graded", "agreed", "fp32_mean", "export_mean")}, + } + + def recipe_fields( recipe, qmode_label: str, @@ -207,7 +253,11 @@ def run( codes: Path | None = None, qmode: str | None = None, allow_base: bool = False, + gate_proxy: Path | None = None, ) -> dict: + """``gate_proxy``: for a Vulkan export with solved codes, the XNNPACK export report that + gates it, since no Vulkan runtime runs on the host. It must have passed its decision gate + on the same source revision, window, codes and output head (check_gate_proxy).""" if backend not in BACKEND_CONFIG: raise ExportError(f"unknown backend {backend!r}; this exporter builds {sorted(BACKEND_CONFIG)}") cfg = settings.load() @@ -256,7 +306,15 @@ def run( f"a {context}-token export needs about {need:,} B with headroom (causal masks alone " f"{sizing.causal_mask_bytes(arch, context):,} B); this host has {budget:,} B" ) - resident = sizing.device_resident_bytes(arch, window, cfg.runtime_overhead_bytes, recipe.qmode) + output_bits = plan.output_bits if recipe.qmode == "8da4w" else 4 + resident = sizing.device_resident_bytes(arch, window, cfg.runtime_overhead_bytes, recipe.qmode, output_bits) + codes_sha256 = file_sha256(codes) if codes is not None else None + proxy = None + if backend == "vulkan" and codes is not None: + # Checked before the export, which takes the better part of an hour at 32k. + proxy = check_gate_proxy(gate_proxy, source.sha, window, codes_sha256, output_bits) + elif gate_proxy is not None: + raise ExportError("a gate proxy stands in for the gate of a Vulkan export with solved codes, and only that") output_repo = naming.output_repo(model_id, cfg.hub_org, cfg.repo_suffix) # The recipe goes in the file name, and a solve is a different recipe from rounding: a @@ -383,8 +441,9 @@ def run( codes is not None, backend, min(cfg.prefill_chunk, window), - output_bits=plan.output_bits if recipe.qmode == "8da4w" else 4, + output_bits=output_bits, ), + **({"codes_sha256": codes_sha256} if codes_sha256 else {}), }, "window": { "context": window, @@ -401,7 +460,7 @@ def run( {"path": pte_path_in_repo, "bytes": pte.stat().st_size, "sha256": sha256(pte)}, ], "estimates": { - "pte_bytes_estimate": sizing.pte_bytes_estimate(arch, window, recipe.qmode), + "pte_bytes_estimate": sizing.pte_bytes_estimate(arch, window, recipe.qmode, output_bits), "pte_bytes_actual": pte.stat().st_size, "export_peak_bytes_estimate": sizing.export_peak_bytes(arch, window), }, @@ -416,6 +475,7 @@ def run( "conversion": conversion, "metadata": method_values, "smoke": smoke_result, + **({"gate_proxy": proxy} if proxy else {}), "run": manifest.run_info(), } (backend_dir / "config.json").write_text( diff --git a/pipeline/publish.py b/pipeline/publish.py index 9ba0736..4f5a3c3 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -62,6 +62,38 @@ def superseded(path: str, folder: str, label: str) -> bool: return window_of(name) == label +def require_publishable(report: dict) -> None: + """Refuse a report that has not passed every check its recipe calls for. + + The workflow runs the smoke test in its own job and fails before reaching publish, but + publishing is also a command a person can run by hand, against a directory exported with + --skip-smoke. Refusing here is what makes "published" mean "ran and answered" rather than + "the workflow happened to be wired correctly that day". A report that was gated must have + passed it. A GPTQ file always has an fp32 reference to be gated against (the solve writes + it), so it must carry a passing gate of its own, or for Vulkan, which the host cannot run, + a gate proxy that passed: a GPTQ report with neither is one whose gate never ran or + crashed (codex review, 2026-11-09). Any other report with no gate (round-to-nearest + Vulkan) is published on its smoke test, as before. + """ + name = report["files"][0]["path"] + smoke = report.get("smoke") + if smoke is None: + raise ValueError( + f"{name} has no smoke result: run `python -m pipeline verify` on this directory before publishing it" + ) + if not smoke.get("passed"): + raise ValueError(f"{name} failed its smoke test: {'; '.join(smoke.get('problems', []))}") + verdict = report.get("gate") + if verdict is not None and not verdict.get("passed"): + raise ValueError(f"{name} failed its decision gate: {'; '.join(verdict.get('problems', []))}") + if (report.get("recipe") or {}).get("int4_codes") == "gptq": + if report.get("backend") == "vulkan": + if not ((report.get("gate_proxy") or {}).get("gate") or {}).get("passed"): + raise ValueError(f"{name} has GPTQ codes and no passing gate proxy (the XNNPACK file from its codes)") + elif verdict is None: + raise ValueError(f"{name} has GPTQ codes and no decision gate: verify it with --gate before publishing") + + def publish_hf( out_dir: Path, backend: str, target: str | None = None, context: int | None = None, attempts: int = 6 ) -> str: @@ -78,25 +110,7 @@ def publish_hf( cfg = settings.load() report = load_report(out_dir, backend, target, context) - # The workflow runs the smoke test in its own job and fails before reaching this, but - # publish is also a command a person can run by hand, against a directory exported with - # --skip-smoke. Refusing here is what makes "published" mean "ran and answered" rather - # than "the workflow happened to be wired correctly that day". - smoke = report.get("smoke") - if smoke is None: - raise ValueError( - f"{report['files'][0]['path']} has no smoke result: run `python -m pipeline verify` " - "on this directory before publishing it" - ) - if not smoke.get("passed"): - raise ValueError(f"{report['files'][0]['path']} failed its smoke test: {'; '.join(smoke.get('problems', []))}") - # A report that was gated must have passed it. One with no gate at all (Vulkan, or a run - # without GPTQ and so without an fp32 reference) is published on its smoke test, as before. - verdict = report.get("gate") - if verdict is not None and not verdict.get("passed"): - raise ValueError( - f"{report['files'][0]['path']} failed its decision gate: {'; '.join(verdict.get('problems', []))}" - ) + require_publishable(report) repo_id = report["output_repo"] folder = _folder(backend, target) label = naming.window_label(report["window"]["context"]) @@ -204,6 +218,7 @@ def publish_release( ) -> str: """Create (or update) a GitHub release with this window's files under the 2 GiB limit.""" report = load_report(out_dir, backend, target, context) + require_publishable(report) folder = out_dir / _folder(backend, target) tag = release_tag(report) label = naming.window_label(report["window"]["context"]) diff --git a/pipeline/sizing.py b/pipeline/sizing.py index 5cc7318..a41145d 100644 --- a/pipeline/sizing.py +++ b/pipeline/sizing.py @@ -113,17 +113,25 @@ def linear_bytes_per_param(qmode: str) -> float: raise ValueError(f"no size model for qmode {qmode!r}; known: {sorted(LINEAR_BYTES_PER_PARAM)}") from None -def pte_bytes_estimate(arch: Architecture, context: int, qmode: str = "8da4w") -> int: +def pte_bytes_estimate(arch: Architecture, context: int, qmode: str = "8da4w", output_bits: int = 4) -> int: + """``output_bits`` 8 prices the output head at one byte per weight (int8 per channel, + XnnpackPlan.output_bits) instead of the int4 rate every other linear is at.""" + per_param = linear_bytes_per_param(qmode) + head = arch.vocab_size * arch.dim + head_extra = head * (EMBEDDING_BYTES_PER_PARAM_INT8 - per_param) if qmode == "8da4w" and output_bits == 8 else 0 raw = ( arch.embedding_params * EMBEDDING_BYTES_PER_PARAM_INT8 - + arch.linear_params * linear_bytes_per_param(qmode) + + arch.linear_params * per_param + + head_extra + context * arch.head_dim * ROPE_TABLE_BYTES_PER_TOKEN_PER_HEAD_DIM ) return int(raw * PTE_OVERHEAD_FACTOR) -def device_resident_bytes(arch: Architecture, context: int, overhead: int, qmode: str = "8da4w") -> int: - return pte_bytes_estimate(arch, context, qmode) + kv_cache_bytes(arch, context) + overhead +def device_resident_bytes( + arch: Architecture, context: int, overhead: int, qmode: str = "8da4w", output_bits: int = 4 +) -> int: + return pte_bytes_estimate(arch, context, qmode, output_bits) + kv_cache_bytes(arch, context) + overhead def causal_mask_bytes(arch: Architecture, context: int) -> int: diff --git a/tests/test_export_xnnpack.py b/tests/test_export_xnnpack.py index 5c40756..2ea4f82 100644 --- a/tests/test_export_xnnpack.py +++ b/tests/test_export_xnnpack.py @@ -110,3 +110,49 @@ def test_an_fp32_report_does_not_claim_int4_weights(): gptq = export_xnnpack.recipe_fields(cfg.xnnpack, "8da4w-gptq", True, "xnnpack", 2048) assert rtn["int4_codes"] == "round-to-nearest" and gptq["int4_codes"] == "gptq" assert "4-bit weights in groups of" in rtn["description"] and rtn["group_size"] == cfg.xnnpack.group_size + + +def _proxy(tmp_path, **changes): + import json + + report = { + "backend": "xnnpack", + "source": {"sha": "a" * 40}, + "window": {"context": 8192}, + "recipe": {"codes_sha256": "c0de", "output_head": "int8 per channel"}, + "gate": {"passed": True, "graded": 8, "agreed": 8, "fp32_mean": 0.66, "export_mean": 0.53}, + "files": [{"path": "xnnpack/x-8k.pte", "sha256": "ff"}], + } + for key, value in changes.items(): + report[key] = value + path = tmp_path / "export-report-8k.json" + path.write_text(json.dumps(report)) + return path + + +def test_a_gate_proxy_counts_only_for_the_same_revision_window_codes_and_head(tmp_path): + from pipeline import export_xnnpack + + ok = export_xnnpack.check_gate_proxy(_proxy(tmp_path), "a" * 40, 8192, "c0de", 8) + assert ok["gate"]["passed"] and ok["file"] == "xnnpack/x-8k.pte" + for changes, args, match in ( + ({}, ("b" * 40, 8192, "c0de", 8), "another source revision"), + ({}, ("a" * 40, 4096, "c0de", 8), "window is 8192"), + ({}, ("a" * 40, 8192, "beef", 8), "other codes"), + ({}, ("a" * 40, 8192, "c0de", 4), "output head differs"), + ({"gate": {"passed": False}}, ("a" * 40, 8192, "c0de", 8), "did not pass"), + ({"backend": "vulkan"}, ("a" * 40, 8192, "c0de", 8), "vulkan report"), + ): + with pytest.raises(export_xnnpack.ExportError, match=match): + export_xnnpack.check_gate_proxy(_proxy(tmp_path, **changes), *args) + with pytest.raises(export_xnnpack.ExportError, match="needs --gate-proxy"): + export_xnnpack.check_gate_proxy(None, "a" * 40, 8192, "c0de", 8) + + +def test_an_int8_head_is_priced_at_a_byte_a_weight(source): + from pipeline import families, sizing + + arch = families.architecture(source("microsoft/Phi-4-mini-instruct").config, 3_836_021_760) + four, eight = sizing.pte_bytes_estimate(arch, 2048), sizing.pte_bytes_estimate(arch, 2048, output_bits=8) + head = 200_064 * 3072 + assert 0.4 * head < eight - four < 0.5 * head # (1 - 0.5625) bytes per weight, plus overhead diff --git a/tests/test_publish.py b/tests/test_publish.py index 2c35a34..1fd2b0b 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -258,3 +258,19 @@ def test_a_report_that_failed_its_decision_gate_is_never_published(tmp_path, mon monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) with pytest.raises(ValueError, match="failed its decision gate: kept fp32's choice on 7 of 8"): publish.publish_hf(out, "xnnpack") + + +def test_a_gptq_file_needs_its_own_gate_or_for_vulkan_a_passing_proxy(): + gptq = {"int4_codes": "gptq", "label": "x", "description": "x"} + cpu = {**report(2048), "recipe": gptq} + with pytest.raises(ValueError, match="no decision gate"): + publish.require_publishable(cpu) + publish.require_publishable({**cpu, "gate": {"passed": True}}) + gpu = {**report(2048, backend="vulkan"), "recipe": gptq} + with pytest.raises(ValueError, match="no passing gate proxy"): + publish.require_publishable(gpu) + with pytest.raises(ValueError, match="no passing gate proxy"): + publish.require_publishable({**gpu, "gate_proxy": {"gate": {"passed": False}}}) + publish.require_publishable({**gpu, "gate_proxy": {"gate": {"passed": True}}}) + # Round to nearest has no fp32 reference to be gated against, as before. + publish.require_publishable({**report(2048, backend="vulkan"), "recipe": {"int4_codes": "round-to-nearest"}}) diff --git a/tests/test_solve.py b/tests/test_solve.py index 0fd5917..e6f64cf 100644 --- a/tests/test_solve.py +++ b/tests/test_solve.py @@ -166,11 +166,34 @@ def __init__(self): model = Tiny() head = model.output.weight.detach().clone() - # The head's int4 codes are dropped rather than injected into an int8 tensor. - codes = {"output": {"qdata": torch.zeros(128, 64, dtype=torch.int8), "scale": torch.ones(128, 2)}} + solved = torch.randint(-8, 8, (64, 64), dtype=torch.int8) + # The head's int4 codes are dropped rather than injected into an int8 tensor; wq's go in. + codes = { + "output": {"qdata": torch.zeros(128, 64, dtype=torch.int8), "scale": torch.ones(128, 2)}, + "wq": {"qdata": solved, "scale": torch.full((64, 2), 0.01)}, + } out = export_with_codes.quantizer(upstream, codes, output_int8=True)(model, "8da4w", group_size=32) w4, w8 = out.wq.weight, out.output.weight - assert int(w4.qdata.min()) >= -8 and int(w4.qdata.max()) <= 7 and tuple(w4.scale.shape) == (64, 2) + assert torch.equal(w4.qdata.to(torch.int8), solved) and tuple(w4.scale.shape) == (64, 2) assert tuple(w8.scale.reshape(128, -1).shape) == (128, 1) # one scale per output channel assert int(w8.qdata.max()) > 7 # the full int8 range, not the int4 grid assert float((w8.dequantize() - head).abs().max()) < float(head.abs().max()) / 100 + + +def test_a_codes_file_that_misses_an_int4_linear_is_refused(): + pytest.importorskip("executorch.examples.models.llama.source_transformation.quantize") + from executorch.examples.models.llama.source_transformation.quantize import quantize as upstream + + from pipeline import export_with_codes + + class Tiny(torch.nn.Module): + def __init__(self): + super().__init__() + self.wq = torch.nn.Linear(64, 64, bias=False) + self.output = torch.nn.Linear(64, 128, bias=False) + + # Only the head's codes: with an int8 head that leaves nothing for wq, which must not be + # exported as "GPTQ" with its weights rounded. + codes = {"output": {"qdata": torch.zeros(128, 64, dtype=torch.int8), "scale": torch.ones(128, 2)}} + with pytest.raises(ValueError, match="no solved codes for 1 int4 linears"): + export_with_codes.quantizer(upstream, codes, output_int8=True)(Tiny(), "8da4w", group_size=32) From 9dbd4432d9e1fa671186177026683074bfb0907e Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 10:58:05 +0800 Subject: [PATCH 05/10] verify --accept-gate: publish past a failed gate only with an operator's recorded reason For Phi-4-mini the operator accepted the 4-bit files with an int8 head, which keep 7 of 8 tool calls. fp32 itself sits near a tie on several of those rows. The measured verdict stays in the report as it is (passed: false), and the reason goes beside it as gate.override. publish accepts a failed gate only when that reason is there, the Vulkan gate proxy carries the reason through, and the README states the shortfall and the reason in the recipe line. No family gets a lower bar: the override is per run, and has to be asked for by name. Co-Authored-By: Claude Opus 5.5 --- pipeline/__main__.py | 10 +++++++++- pipeline/export_xnnpack.py | 4 ++-- pipeline/manifest.py | 7 +++++++ pipeline/publish.py | 5 +++-- pipeline/verify.py | 21 +++++++++++++++++---- tests/test_publish.py | 13 +++++++++++++ tests/test_verify.py | 38 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 9 deletions(-) diff --git a/pipeline/__main__.py b/pipeline/__main__.py index 040ba74..142ad72 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -60,7 +60,9 @@ def _verify(args) -> int: try: gate = Path(args.gate) if args.gate else None - summary = verify_module.run(Path(args.out), args.backend, Path(args.work), gate_reference=gate) + summary = verify_module.run( + Path(args.out), args.backend, Path(args.work), gate_reference=gate, accept_gate=args.accept_gate + ) except ExportError as error: print(f"verify failed: {error}", file=sys.stderr) return 2 @@ -448,6 +450,12 @@ def main(argv: list[str] | None = None) -> int: default=None, help="gate.json from `solve`: the file must decide like the fp32 model before it publishes", ) + verify.add_argument( + "--accept-gate", + default=None, + metavar="REASON", + help="publish even if the gate fails, recording this reason beside the measured verdict", + ) verify.set_defaults(func=_verify) audit = commands.add_parser("audit", help="gate a file that is already published, without rebuilding it") diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 1557b15..4efa95c 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -172,7 +172,7 @@ def check_gate_proxy(path: Path | None, sha: str, window: int, codes_sha256: str problems.append("it was built from other codes") if (recipe.get("output_head") == "int8 per channel") != (output_bits == 8): problems.append("its output head differs") - if not verdict.get("passed"): + if not verdict.get("passed") and not (verdict.get("override") or {}).get("reason"): problems.append("its decision gate did not pass" if verdict else "it has no decision gate") if problems: raise ExportError(f"gate proxy {path} does not stand for this export: {'; '.join(problems)}") @@ -180,7 +180,7 @@ def check_gate_proxy(path: Path | None, sha: str, window: int, codes_sha256: str "report": Path(path).name, "file": report["files"][0]["path"], "sha256": report["files"][0].get("sha256"), - "gate": {k: verdict.get(k) for k in ("passed", "graded", "agreed", "fp32_mean", "export_mean")}, + "gate": {k: verdict.get(k) for k in ("passed", "graded", "agreed", "fp32_mean", "export_mean", "override")}, } diff --git a/pipeline/manifest.py b/pipeline/manifest.py index cc78886..1915251 100644 --- a/pipeline/manifest.py +++ b/pipeline/manifest.py @@ -265,6 +265,13 @@ def readme(repo_id: str, reports: list[dict], hub_tags: list[str], license_files continue described.add(key) line = f"- {BACKEND_TITLES[r['backend']]} {target(r)}: {recipe['description']}" + verdict = r.get("gate") or (r.get("gate_proxy") or {}).get("gate") or {} + if (verdict.get("override") or {}).get("reason"): + line += ( + f" Published past its decision gate: it kept fp32's choice on {verdict.get('agreed')} of " + f"{verdict.get('graded')} tool-call rows, under the 90% the gate asks, and was accepted because " + f"{verdict['override']['reason']}" + ) same = [x for x in reports if (x["backend"], x.get("target"), x["recipe"]["description"]) == key] runs = sorted({x["run"]["url"] for x in same if x.get("run", {}).get("url")}) if runs: diff --git a/pipeline/publish.py b/pipeline/publish.py index 4f5a3c3..32f7689 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -84,11 +84,12 @@ def require_publishable(report: dict) -> None: if not smoke.get("passed"): raise ValueError(f"{name} failed its smoke test: {'; '.join(smoke.get('problems', []))}") verdict = report.get("gate") - if verdict is not None and not verdict.get("passed"): + if verdict is not None and not verdict.get("passed") and not (verdict.get("override") or {}).get("reason"): raise ValueError(f"{name} failed its decision gate: {'; '.join(verdict.get('problems', []))}") if (report.get("recipe") or {}).get("int4_codes") == "gptq": if report.get("backend") == "vulkan": - if not ((report.get("gate_proxy") or {}).get("gate") or {}).get("passed"): + proxy = (report.get("gate_proxy") or {}).get("gate") or {} + if not proxy.get("passed") and not (proxy.get("override") or {}).get("reason"): raise ValueError(f"{name} has GPTQ codes and no passing gate proxy (the XNNPACK file from its codes)") elif verdict is None: raise ValueError(f"{name} has GPTQ codes and no decision gate: verify it with --gate before publishing") diff --git a/pipeline/verify.py b/pipeline/verify.py index 4d1ab94..dc0638e 100644 --- a/pipeline/verify.py +++ b/pipeline/verify.py @@ -30,8 +30,17 @@ def reports(out_dir: Path, backend: str) -> list[Path]: return found -def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None = None) -> dict: - """Run the smoke test for every window in ``out_dir`` and write the results back.""" +def run( + out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None = None, accept_gate: str | None = None +) -> dict: + """Run the smoke test for every window in ``out_dir`` and write the results back. + + ``accept_gate`` is an operator's reason for publishing a file that fails its decision + gate. The verdict is kept as measured (``passed`` stays false) and the reason is recorded + beside it as ``override``; publish accepts a failed gate only with one. It exists for a + model whose fp32 decisions are themselves near ties, so that a 4-bit file cannot keep all + of them: Phi-4-mini, 7 of 8 kept, 2026-11-09. + """ results = [] gates: list[dict] = [] for path in reports(out_dir, backend): @@ -40,7 +49,8 @@ def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None # first run that passed the smoke test and then failed (or never reached) the gate # leaves a smoke result behind, and skipping on that alone let a second run report # success with nothing gated (codex review, 2026-11-09). - gated = gate_reference is None or (report.get("gate") or {}).get("passed") + recorded = report.get("gate") or {} + gated = gate_reference is None or recorded.get("passed") or (accept_gate and recorded.get("override")) if report.get("smoke") is not None and gated: print(f"==> {path.name}: already verified, leaving it") results.append(report["smoke"]) @@ -101,6 +111,9 @@ def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None "the quiet ones, so the gate checked that the export tracks its fp32 model " "and not that it still calls tools" ) + if not verdict["passed"] and accept_gate: + verdict["override"] = {"reason": accept_gate} + print(f" gate failed and accepted by the operator: {accept_gate}") report["gate"] = verdict gates.append(verdict) @@ -111,7 +124,7 @@ def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None if failed: problems = "; ".join(p for r in failed for p in r.get("problems", [])) raise ExportError(f"smoke test failed: {problems}") - regressed = [g for g in gates if not g.get("passed")] + regressed = [g for g in gates if not g.get("passed") and not g.get("override")] if regressed: problems = "; ".join(p for g in regressed for p in g.get("problems", [])) raise ExportError(f"decision gate failed: {problems}") diff --git a/tests/test_publish.py b/tests/test_publish.py index 1fd2b0b..d376d06 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -274,3 +274,16 @@ def test_a_gptq_file_needs_its_own_gate_or_for_vulkan_a_passing_proxy(): publish.require_publishable({**gpu, "gate_proxy": {"gate": {"passed": True}}}) # Round to nearest has no fp32 reference to be gated against, as before. publish.require_publishable({**report(2048, backend="vulkan"), "recipe": {"int4_codes": "round-to-nearest"}}) + + +def test_a_failed_gate_publishes_only_with_an_operators_recorded_reason(): + gptq = {"int4_codes": "gptq", "label": "x", "description": "x"} + failed = {"passed": False, "agreed": 7, "graded": 8, "problems": ["7 of 8"]} + cpu = {**report(2048), "recipe": gptq, "gate": failed} + with pytest.raises(ValueError, match="failed its decision gate"): + publish.require_publishable(cpu) + with pytest.raises(ValueError, match="failed its decision gate"): + publish.require_publishable({**cpu, "gate": {**failed, "override": {"reason": ""}}}) + publish.require_publishable({**cpu, "gate": {**failed, "override": {"reason": "fp32 is near a tie"}}}) + gpu = {**report(2048, backend="vulkan"), "recipe": gptq} + publish.require_publishable({**gpu, "gate_proxy": {"gate": {**failed, "override": {"reason": "near a tie"}}}}) diff --git a/tests/test_verify.py b/tests/test_verify.py index 8c04f60..dccaaf9 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -50,3 +50,41 @@ def test_a_report_that_passed_both_is_left_alone(tmp_path, reaches_smoke): assert verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference)["passed"] out_ungated = write(tmp_path / "b", {"smoke": {"passed": True}}) assert verify.run(out_ungated, "xnnpack", tmp_path / "work")["passed"] + + +def test_an_accepted_gate_keeps_its_measured_verdict_and_records_the_reason(tmp_path, monkeypatch): + from pipeline import gate, verify + + out = write(tmp_path, {}) + reference = tmp_path / "gate.json" + reference.write_text("{}") + monkeypatch.setattr( + verify.hub, + "fetch", + lambda *a, **k: type("S", (), {"config": {}, "total_params": 1, "tokenizer_config": None})(), + ) + monkeypatch.setattr(verify.hub, "download", lambda *a, **k: None) + monkeypatch.setattr(verify.families, "architecture", lambda *a, **k: type("A", (), {"total_params": 1})()) + monkeypatch.setattr(verify.smoke, "run", lambda *a, **k: {"passed": True}) + monkeypatch.setattr(gate, "measure", lambda *a, **k: {}) + monkeypatch.setattr( + gate, + "check", + lambda *a, **k: { + "passed": False, + "problems": ["7 of 8"], + "agreed": 7, + "graded": 8, + "fp32_mean": 0.66, + "export_mean": 0.51, + }, + ) + import transformers + + monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: None) + with pytest.raises(verify.ExportError, match="decision gate failed"): + verify.run(out, "xnnpack", tmp_path / "w1", gate_reference=reference) + summary = verify.run(out, "xnnpack", tmp_path / "w2", gate_reference=reference, accept_gate="near a tie") + assert summary["passed"] + saved = json.loads((out / "xnnpack" / "export-report-2k.json").read_text())["gate"] + assert saved["passed"] is False and saved["override"] == {"reason": "near a tie"} From 767ad6513957341b276fc3507dfeec69f53b006b Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 11:01:15 +0800 Subject: [PATCH 06/10] Tie every verdict to the bytes it measured, and a gate proxy to the whole build Codex review, round four: - publish-hf and publish-release refuse a file whose sha256 is not the one its report records. verify re-checks a report whose file was rebuilt rather than skipping it on its old verdict. - An export with codes reads them from its own copy, the one it hashes. A shared codes.pt replaced between the hash and the read can no longer leave the report naming other codes. - An empty codes file is refused instead of exporting every linear rounded under a GPTQ label. - A gate proxy has to match the source repo, the ExecuTorch version and every recipe field that shapes the graph or the weights: model class, converter, params, qmode, group size, embedding quantization, prefill chunk and output head. It used to match only the revision, window, codes and head. A proxy report that cannot be read is an ExportError. - The window table prices the int8 head too, not only the chosen window. Co-Authored-By: Claude Opus 5.5 --- pipeline/export_with_codes.py | 5 +- pipeline/export_xnnpack.py | 115 ++++++++++++++++++++++------------ pipeline/publish.py | 14 +++++ pipeline/sizing.py | 3 +- pipeline/verify.py | 7 ++- tests/test_export_xnnpack.py | 55 ++++++++++++---- tests/test_publish.py | 21 ++++++- tests/test_verify.py | 10 ++- 8 files changed, 172 insertions(+), 58 deletions(-) diff --git a/pipeline/export_with_codes.py b/pipeline/export_with_codes.py index 98fae77..ffb36a1 100644 --- a/pipeline/export_with_codes.py +++ b/pipeline/export_with_codes.py @@ -68,7 +68,7 @@ def quantizer(upstream, codes: dict, output_int8: bool): and for ``output_int8`` the output head at int8 per channel rather than int4.""" import torch - codes_given = bool(codes) + codes_given = bool(codes) # main() refuses an empty codes file before this if output_int8: # The head is not int4 here, so its int4 codes have nowhere to go. codes = {k: v for k, v in codes.items() if k != "output"} @@ -157,6 +157,9 @@ def main(argv: list[str] | None = None) -> int: # weights_only: the codes file is an artifact passed between CI jobs, and it holds # nothing but tensors, so there is no reason to let it unpickle arbitrary objects. codes = torch.load(args.codes, map_location="cpu", weights_only=True) if args.codes else {} + if args.codes and not codes: + # An empty codes file would export every linear rounded under a GPTQ label. + raise ValueError(f"{args.codes} holds no codes") quantize_module.quantize = quantizer(quantize_module.quantize, codes, args.output_int8) from executorch.extension.llm.export.export_llm import main as export_llm_main diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 4efa95c..2a557ea 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -138,50 +138,53 @@ def pick_runner( return None -def file_sha256(path: Path) -> str: - import hashlib - - digest = hashlib.sha256() - with open(path, "rb") as f: - for block in iter(lambda: f.read(1 << 24), b""): - digest.update(block) - return digest.hexdigest() - - -def check_gate_proxy(path: Path | None, sha: str, window: int, codes_sha256: str, output_bits: int) -> dict: +def check_gate_proxy(path: Path | None, sha: str, window: int, codes_sha256: str, expected: dict) -> dict: """What stands in for a Vulkan file's gate: the XNNPACK report built from the same codes. The two delegates run the same torchao linears with the same codes, so the XNNPACK file - is the closest measurement of the Vulkan one the host can make. It counts only for the - same source revision, window (LongRoPE picks its factors by window), codes file and output - head, and only when its own gate passed; anything else would label an ungated file as - gated (codex review, 2026-11-09). + is the closest measurement of the Vulkan one the host can make. It counts only when it + was built the same way: the same source repo and revision, window (LongRoPE picks its + factors by window), codes file, toolchain and every recipe field that shapes the weights + or the graph (``expected``: model class, converter, params, qmode, group size, embedding + quantization, prefill chunk, output head). And only when its own gate passed, or was + accepted by an operator with a recorded reason. Anything else would label an ungated file + as gated (codex review, 2026-11-09). """ if path is None: raise ExportError("a Vulkan export with solved codes needs --gate-proxy, the XNNPACK report that gates it") - report = json.loads(Path(path).read_text()) - recipe, verdict = report.get("recipe") or {}, report.get("gate") or {} - problems = [] - if report.get("backend") != "xnnpack": - problems.append(f"it is a {report.get('backend')} report") - if (report.get("source") or {}).get("sha") != sha: - problems.append("it was built from another source revision") - if (report.get("window") or {}).get("context") != window: - problems.append(f"its window is {(report.get('window') or {}).get('context')}, not {window}") - if recipe.get("codes_sha256") != codes_sha256: - problems.append("it was built from other codes") - if (recipe.get("output_head") == "int8 per channel") != (output_bits == 8): - problems.append("its output head differs") - if not verdict.get("passed") and not (verdict.get("override") or {}).get("reason"): - problems.append("its decision gate did not pass" if verdict else "it has no decision gate") - if problems: - raise ExportError(f"gate proxy {path} does not stand for this export: {'; '.join(problems)}") - return { - "report": Path(path).name, - "file": report["files"][0]["path"], - "sha256": report["files"][0].get("sha256"), - "gate": {k: verdict.get(k) for k in ("passed", "graded", "agreed", "fp32_mean", "export_mean", "override")}, - } + try: + report = json.loads(Path(path).read_text()) + recipe, verdict = report.get("recipe") or {}, report.get("gate") or {} + problems = [] + if report.get("backend") != "xnnpack": + problems.append(f"it is a {report.get('backend')} report") + if (report.get("source") or {}).get("id") != expected["source_id"]: + problems.append("it was built from another source repo") + if (report.get("source") or {}).get("sha") != sha: + problems.append("it was built from another source revision") + if (report.get("window") or {}).get("context") != window: + problems.append(f"its window is {(report.get('window') or {}).get('context')}, not {window}") + if recipe.get("codes_sha256") != codes_sha256: + problems.append("it was built from other codes") + if (report.get("toolchain") or {}).get("executorch") != expected["executorch"]: + problems.append("it was built with another ExecuTorch") + for key, value in expected["recipe"].items(): + if recipe.get(key) != value: + problems.append(f"its {key} differs") + if not verdict.get("passed") and not (verdict.get("override") or {}).get("reason"): + problems.append("its decision gate did not pass" if verdict else "it has no decision gate") + if problems: + raise ExportError(f"gate proxy {path} does not stand for this export: {'; '.join(problems)}") + return { + "report": Path(path).name, + "file": report["files"][0]["path"], + "sha256": report["files"][0]["sha256"], + "gate": {k: verdict.get(k) for k in ("passed", "graded", "agreed", "fp32_mean", "export_mean", "override")}, + } + except ExportError: + raise + except (OSError, ValueError, KeyError, IndexError, TypeError) as error: + raise ExportError(f"gate proxy {path} could not be read: {error!r}") from error def recipe_fields( @@ -283,7 +286,14 @@ def run( raise ExportError("solved codes are int4; a GPTQ run exports 8da4w") recipe = dataclasses.replace(recipe, qmode=qmode) choice = sizing.choose_context( - arch, cfg.context_tiers, cfg.device_budget_bytes, cfg.runtime_overhead_bytes, budget, backend, recipe.qmode + arch, + cfg.context_tiers, + cfg.device_budget_bytes, + cfg.runtime_overhead_bytes, + budget, + backend, + recipe.qmode, + plan.output_bits if recipe.qmode == "8da4w" else 4, ) # Every tier is exported, one job per window; the app and the benchmarker decide what # fits a phone (the estimate goes into the report as fits_phone_budget). Only the host @@ -308,11 +318,34 @@ def run( ) output_bits = plan.output_bits if recipe.qmode == "8da4w" else 4 resident = sizing.device_resident_bytes(arch, window, cfg.runtime_overhead_bytes, recipe.qmode, output_bits) - codes_sha256 = file_sha256(codes) if codes is not None else None + codes_sha256 = None + if codes is not None: + # The export reads the codes from a copy of its own, hashed here: a shared codes.pt + # replaced between the hash and the read would otherwise leave the report naming + # codes the file was not built from (codex review, 2026-11-09). + work_dir.mkdir(parents=True, exist_ok=True) + private = work_dir / "codes.pt" + shutil.copyfile(codes, private) + codes, codes_sha256 = private, sha256(private) proxy = None if backend == "vulkan" and codes is not None: # Checked before the export, which takes the better part of an hour at 32k. - proxy = check_gate_proxy(gate_proxy, source.sha, window, codes_sha256, output_bits) + expected = { + "source_id": source.id, + "executorch": toolchain()["executorch"], + "recipe": { + "model_class": plan.model_class, + "converter": plan.converter, + "params": plan.params, + "export_llm_qmode": QMODES[recipe.qmode], + "embedding_hqq": recipe.embedding_hqq, + "embedding_quantize": recipe.embedding_quantize, + "group_size": recipe.group_size, + "prefill_chunk": min(cfg.prefill_chunk, window), + "output_head": "int8 per channel" if output_bits == 8 else None, + }, + } + proxy = check_gate_proxy(gate_proxy, source.sha, window, codes_sha256, expected) elif gate_proxy is not None: raise ExportError("a gate proxy stands in for the gate of a Vulkan export with solved codes, and only that") diff --git a/pipeline/publish.py b/pipeline/publish.py index 32f7689..b413397 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -62,6 +62,18 @@ def superseded(path: str, folder: str, label: str) -> bool: return window_of(name) == label +def require_same_bytes(out_dir: Path, report: dict) -> None: + """Every file the report lists is, byte for byte, the file it describes: a verdict is + about the bytes it measured, and a .pte rebuilt or replaced after verify is not that file + (codex review, 2026-11-09).""" + for entry in report["files"]: + local = out_dir / entry["path"] + if not local.is_file(): + raise ValueError(f"{entry['path']} is listed in the report but not in {out_dir}") + if exporting.sha256(local) != entry.get("sha256"): + raise ValueError(f"{entry['path']} is not the file its report describes (sha256 differs)") + + def require_publishable(report: dict) -> None: """Refuse a report that has not passed every check its recipe calls for. @@ -126,6 +138,7 @@ def publish_hf( # A tokenizer inside any backend folder makes the app stop lending the root one to # every other folder (HuggingFaceClient.tokenizerFor), breaking the other backends. raise ValueError(f"tokenizer files must stay at the repo root, found {scoped}") + require_same_bytes(out_dir, report) hf = hub.api() hf.create_repo(repo_id, repo_type="model", exist_ok=True) @@ -220,6 +233,7 @@ def publish_release( """Create (or update) a GitHub release with this window's files under the 2 GiB limit.""" report = load_report(out_dir, backend, target, context) require_publishable(report) + require_same_bytes(out_dir, report) folder = out_dir / _folder(backend, target) tag = release_tag(report) label = naming.window_label(report["window"]["context"]) diff --git a/pipeline/sizing.py b/pipeline/sizing.py index a41145d..22c8f1e 100644 --- a/pipeline/sizing.py +++ b/pipeline/sizing.py @@ -209,13 +209,14 @@ def choose_context( host_budget: int | None, backend: str = "xnnpack", qmode: str = "8da4w", + output_bits: int = 4, ) -> WindowChoice: """Largest tier whose phone residency and host export peak both fit. `qmode` sizes the linears: an fp32 file holds them at four bytes a weight, eight times an 8da4w one.""" rows = [] chosen = None for context in sorted(tiers, reverse=True): - resident = device_resident_bytes(arch, context, runtime_overhead, qmode) + resident = device_resident_bytes(arch, context, runtime_overhead, qmode, output_bits) peak = export_peak_bytes(arch, context) fits_device = resident <= device_budget fits_host = host_budget is None or host_need_bytes(arch, context, backend) <= host_budget diff --git a/pipeline/verify.py b/pipeline/verify.py index dc0638e..395f892 100644 --- a/pipeline/verify.py +++ b/pipeline/verify.py @@ -19,7 +19,7 @@ import json from pathlib import Path -from pipeline import families, gate, hub, settings, smoke +from pipeline import exporting, families, gate, hub, settings, smoke from pipeline.exporting import ExportError @@ -51,7 +51,10 @@ def run( # success with nothing gated (codex review, 2026-11-09). recorded = report.get("gate") or {} gated = gate_reference is None or recorded.get("passed") or (accept_gate and recorded.get("override")) - if report.get("smoke") is not None and gated: + # And only while the file is the one that was verified: a rebuilt .pte is a new file. + listed = out_dir / report["files"][0]["path"] + same = listed.is_file() and exporting.sha256(listed) == report["files"][0].get("sha256") + if report.get("smoke") is not None and gated and same: print(f"==> {path.name}: already verified, leaving it") results.append(report["smoke"]) continue diff --git a/tests/test_export_xnnpack.py b/tests/test_export_xnnpack.py index 2ea4f82..ce8cd19 100644 --- a/tests/test_export_xnnpack.py +++ b/tests/test_export_xnnpack.py @@ -112,14 +112,29 @@ def test_an_fp32_report_does_not_claim_int4_weights(): assert "4-bit weights in groups of" in rtn["description"] and rtn["group_size"] == cfg.xnnpack.group_size +RECIPE = { + "model_class": "phi_4_mini", + "converter": "phi4", + "params": {"dim": 3072}, + "export_llm_qmode": "8da4w", + "embedding_hqq": False, + "embedding_quantize": "8,0", + "group_size": 32, + "prefill_chunk": 2048, + "output_head": "int8 per channel", +} +EXPECTED = {"source_id": "microsoft/Phi-4-mini-instruct", "executorch": "1.5.1", "recipe": RECIPE} + + def _proxy(tmp_path, **changes): import json report = { "backend": "xnnpack", - "source": {"sha": "a" * 40}, + "source": {"id": "microsoft/Phi-4-mini-instruct", "sha": "a" * 40}, + "toolchain": {"executorch": "1.5.1"}, "window": {"context": 8192}, - "recipe": {"codes_sha256": "c0de", "output_head": "int8 per channel"}, + "recipe": {**RECIPE, "codes_sha256": "c0de"}, "gate": {"passed": True, "graded": 8, "agreed": 8, "fp32_mean": 0.66, "export_mean": 0.53}, "files": [{"path": "xnnpack/x-8k.pte", "sha256": "ff"}], } @@ -130,23 +145,39 @@ def _proxy(tmp_path, **changes): return path -def test_a_gate_proxy_counts_only_for_the_same_revision_window_codes_and_head(tmp_path): +def test_a_gate_proxy_counts_only_for_the_same_build(tmp_path): + import copy + from pipeline import export_xnnpack - ok = export_xnnpack.check_gate_proxy(_proxy(tmp_path), "a" * 40, 8192, "c0de", 8) + ok = export_xnnpack.check_gate_proxy(_proxy(tmp_path), "a" * 40, 8192, "c0de", EXPECTED) assert ok["gate"]["passed"] and ok["file"] == "xnnpack/x-8k.pte" + four_bit_head = copy.deepcopy(EXPECTED) + four_bit_head["recipe"]["output_head"] = None + other_repo = {**EXPECTED, "source_id": "microsoft/Phi-4-mini-reasoning"} + other_toolchain = {**EXPECTED, "executorch": "1.6.0"} for changes, args, match in ( - ({}, ("b" * 40, 8192, "c0de", 8), "another source revision"), - ({}, ("a" * 40, 4096, "c0de", 8), "window is 8192"), - ({}, ("a" * 40, 8192, "beef", 8), "other codes"), - ({}, ("a" * 40, 8192, "c0de", 4), "output head differs"), - ({"gate": {"passed": False}}, ("a" * 40, 8192, "c0de", 8), "did not pass"), - ({"backend": "vulkan"}, ("a" * 40, 8192, "c0de", 8), "vulkan report"), + ({}, ("b" * 40, 8192, "c0de", EXPECTED), "another source revision"), + ({}, ("a" * 40, 4096, "c0de", EXPECTED), "window is 8192"), + ({}, ("a" * 40, 8192, "beef", EXPECTED), "other codes"), + ({}, ("a" * 40, 8192, "c0de", four_bit_head), "output_head differs"), + ({}, ("a" * 40, 8192, "c0de", other_repo), "another source repo"), + ({}, ("a" * 40, 8192, "c0de", other_toolchain), "another ExecuTorch"), + ( + {"recipe": {**RECIPE, "codes_sha256": "c0de", "embedding_quantize": "4,32"}}, + ("a" * 40, 8192, "c0de", EXPECTED), + "embedding_quantize differs", + ), + ({"gate": {"passed": False}}, ("a" * 40, 8192, "c0de", EXPECTED), "did not pass"), + ({"backend": "vulkan"}, ("a" * 40, 8192, "c0de", EXPECTED), "vulkan report"), + ({"files": []}, ("a" * 40, 8192, "c0de", EXPECTED), "could not be read"), ): with pytest.raises(export_xnnpack.ExportError, match=match): export_xnnpack.check_gate_proxy(_proxy(tmp_path, **changes), *args) with pytest.raises(export_xnnpack.ExportError, match="needs --gate-proxy"): - export_xnnpack.check_gate_proxy(None, "a" * 40, 8192, "c0de", 8) + export_xnnpack.check_gate_proxy(None, "a" * 40, 8192, "c0de", EXPECTED) + accepted = {"passed": False, "agreed": 7, "graded": 8, "override": {"reason": "near a tie"}} + assert export_xnnpack.check_gate_proxy(_proxy(tmp_path, gate=accepted), "a" * 40, 8192, "c0de", EXPECTED) def test_an_int8_head_is_priced_at_a_byte_a_weight(source): @@ -156,3 +187,5 @@ def test_an_int8_head_is_priced_at_a_byte_a_weight(source): four, eight = sizing.pte_bytes_estimate(arch, 2048), sizing.pte_bytes_estimate(arch, 2048, output_bits=8) head = 200_064 * 3072 assert 0.4 * head < eight - four < 0.5 * head # (1 - 0.5625) bytes per weight, plus overhead + rows = sizing.choose_context(arch, (2048,), 5_000_000_000, 500_000_000, None, output_bits=8).table + assert rows[0]["device_resident_bytes"] == sizing.device_resident_bytes(arch, 2048, 500_000_000, output_bits=8) diff --git a/tests/test_publish.py b/tests/test_publish.py index d376d06..fed1404 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1,5 +1,6 @@ """Publishing one window into a repo that already holds other windows and backends.""" +import hashlib import json from types import SimpleNamespace @@ -20,7 +21,14 @@ def report(context, backend="xnnpack", target=None, sha="abcdef0123456789", run= "toolchain": {"executorch": "1.4.0"}, "recipe": {"label": "8da4w-g32, int8 embeddings", "description": "recipe text."}, "window": {"context": context, "kv_cache_bytes_per_token": 229_376, "fits_phone_budget": context <= 8192}, - "files": [{"path": f"{folder}/Qwen3-1.7B-8da4w-{label}.pte", "bytes": 1_000 * context, "sha256": "ff"}], + # The tests write b"pte" as the file; publish checks the bytes against this. + "files": [ + { + "path": f"{folder}/Qwen3-1.7B-8da4w-{label}.pte", + "bytes": 1_000 * context, + "sha256": hashlib.sha256(b"pte").hexdigest(), + } + ], "metadata": {"get_max_context_len": context, "get_max_seq_len": min(2048, context)}, "smoke": {"passed": True, "answered": True}, "run": {"url": run}, @@ -287,3 +295,14 @@ def test_a_failed_gate_publishes_only_with_an_operators_recorded_reason(): publish.require_publishable({**cpu, "gate": {**failed, "override": {"reason": "fp32 is near a tie"}}}) gpu = {**report(2048, backend="vulkan"), "recipe": gptq} publish.require_publishable({**gpu, "gate_proxy": {"gate": {**failed, "override": {"reason": "near a tie"}}}}) + + +def test_a_file_that_is_not_the_one_its_report_describes_is_never_published(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"rebuilt after verify") + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(report(2048))) + monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) + with pytest.raises(ValueError, match="not the file its report describes"): + publish.publish_hf(out, "xnnpack") diff --git a/tests/test_verify.py b/tests/test_verify.py index dccaaf9..c5786a1 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -1,5 +1,6 @@ """verify decides what still needs checking from the report it finds.""" +import hashlib import json import pytest @@ -17,7 +18,7 @@ def write(tmp_path, report): (out / "xnnpack" / "x-2k.pte").write_bytes(b"pte") base = { "window": {"context": 2048}, - "files": [{"path": "xnnpack/x-2k.pte"}], + "files": [{"path": "xnnpack/x-2k.pte", "sha256": hashlib.sha256(b"pte").hexdigest()}], "tokenizer": "tokenizer.json", "source": {"id": "org/x", "sha": "0" * 40, "variant": "instruct"}, } @@ -88,3 +89,10 @@ def test_an_accepted_gate_keeps_its_measured_verdict_and_records_the_reason(tmp_ assert summary["passed"] saved = json.loads((out / "xnnpack" / "export-report-2k.json").read_text())["gate"] assert saved["passed"] is False and saved["override"] == {"reason": "near a tie"} + + +def test_a_verified_report_is_verified_again_when_its_file_was_rebuilt(tmp_path, reaches_smoke): + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True}}) + (out / "xnnpack" / "x-2k.pte").write_bytes(b"rebuilt") + with pytest.raises(Ran): + verify.run(out, "xnnpack", tmp_path / "work") From 489d555b76f674321185fe958e15e7f4d838e268 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 11:21:52 +0800 Subject: [PATCH 07/10] README states every overridden window; publish uploads only files its report vouches for Codex review, round five: - The README's recipe line names each window published past its gate, with its own agreement count and reason, instead of taking the first report's verdict for all of them. - publish-hf refuses a model file in the folder that the report does not list. An extra or renamed .pte had no verdict and no digest behind it. - The export reads its private copy of the codes by absolute path, since its subprocess runs inside the work directory. Co-Authored-By: Claude Opus 5.5 --- pipeline/export_xnnpack.py | 3 ++- pipeline/manifest.py | 20 ++++++++++++++------ pipeline/publish.py | 6 ++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 2a557ea..71d66ee 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -324,7 +324,8 @@ def run( # replaced between the hash and the read would otherwise leave the report naming # codes the file was not built from (codex review, 2026-11-09). work_dir.mkdir(parents=True, exist_ok=True) - private = work_dir / "codes.pt" + # Absolute: the export's subprocess runs inside work_dir (codex review, round five). + private = (work_dir / "codes.pt").resolve() shutil.copyfile(codes, private) codes, codes_sha256 = private, sha256(private) proxy = None diff --git a/pipeline/manifest.py b/pipeline/manifest.py index 1915251..5935517 100644 --- a/pipeline/manifest.py +++ b/pipeline/manifest.py @@ -265,14 +265,22 @@ def readme(repo_id: str, reports: list[dict], hub_tags: list[str], license_files continue described.add(key) line = f"- {BACKEND_TITLES[r['backend']]} {target(r)}: {recipe['description']}" - verdict = r.get("gate") or (r.get("gate_proxy") or {}).get("gate") or {} - if (verdict.get("override") or {}).get("reason"): + same = [x for x in reports if (x["backend"], x.get("target"), x["recipe"]["description"]) == key] + # Every window of this recipe that went out past its gate, each with its own numbers: + # one window's verdict cannot speak for the others (codex review, round five). + accepted = [] + for x in sorted(same, key=lambda x: x["window"]["context"]): + verdict = x.get("gate") or (x.get("gate_proxy") or {}).get("gate") or {} + reason = (verdict.get("override") or {}).get("reason") + if reason: + accepted.append((naming.window_label(x["window"]["context"]), verdict, reason)) + if accepted: + windows = ", ".join(f"{label} kept {v.get('agreed')} of {v.get('graded')}" for label, v, _ in accepted) + reasons = sorted({reason for _, _, reason in accepted}) line += ( - f" Published past its decision gate: it kept fp32's choice on {verdict.get('agreed')} of " - f"{verdict.get('graded')} tool-call rows, under the 90% the gate asks, and was accepted because " - f"{verdict['override']['reason']}" + f" Published past its decision gate, which asks that 90% of fp32's tool calls be kept: {windows}. " + f"Accepted because {' / '.join(reasons)}" ) - same = [x for x in reports if (x["backend"], x.get("target"), x["recipe"]["description"]) == key] runs = sorted({x["run"]["url"] for x in same if x.get("run", {}).get("url")}) if runs: line += " Built by " + ", ".join(f"[run {i + 1}]({url})" for i, url in enumerate(runs)) + "." diff --git a/pipeline/publish.py b/pipeline/publish.py index b413397..0d32cda 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -138,6 +138,12 @@ def publish_hf( # A tokenizer inside any backend folder makes the app stop lending the root one to # every other folder (HuggingFaceClient.tokenizerFor), breaking the other backends. raise ValueError(f"tokenizer files must stay at the repo root, found {scoped}") + # Only what the report vouches for: a model file in the folder that the report does not + # list (an extra or renamed .pte) has no verdict and no digest (codex review, round five). + listed = {entry["path"] for entry in report["files"]} + unvouched = [path for path in new_paths if path.endswith((".pte", ".bin")) and path not in listed] + if unvouched: + raise ValueError(f"{unvouched} are in {folder} but not in its report; nothing vouches for them") require_same_bytes(out_dir, report) hf = hub.api() From 76380c2bee6325dfdf204f4924dd704d915b2841 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 12:20:56 +0800 Subject: [PATCH 08/10] PLAN: Phi-4-mini's recipe and gate override, Vulkan GPTQ through a gate proxy, the thinking-only gate gap What the 2026-11-09 run decided and measured, so the next export does not rediscover it: - Phi-4-mini matches fp32 before quantization. Its windows of 8k and up run LongRoPE's long factors from position 0. It keeps 7 of 8 tool calls at 4 bits and was published through an operator override. - Vulkan takes GPTQ codes only with an XNNPACK twin as its gate. - The one-token gate cannot see Qwen3-4B-Thinking-2507's decision. - 32k at 4B needs about 143 GB. Co-Authored-By: Claude Opus 5.5 --- docs/PLAN.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/PLAN.md b/docs/PLAN.md index 9afeb07..138a9ae 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -131,6 +131,17 @@ so the app's `CompiledBackend.of` reads "vulkan". The Linux wheel's runner has n kernels and the runner no GPU, so the check is structural (`smoke.structural_check`: `forward` delegates to `VulkanBackend`); quality and speed are measured on the devices. +- **GPTQ codes, gated through the XNNPACK twin** (2026-11-09): `export-vulkan --codes + --gate-proxy .json>` puts the solve's codes into the + same torchao linears, as for XNNPACK. With no Vulkan runtime on the host, the XNNPACK file + built from the same codes is the gate. `export_xnnpack.check_gate_proxy` accepts it only + when it was built the same way: source repo and revision, window, codes sha256, + ExecuTorch version, and every recipe field that shapes the graph or the weights. Its own + gate must also have passed or been accepted. The proxy is recorded in the Vulkan report, + and `publish` refuses a GPTQ Vulkan file without one. Why: round to nearest kept none of + Phi-4-mini's 8 graded tool calls (mean 0.05 against fp32's 0.66, measured on an XNNPACK + file with the same quantization). The `export-vulkan` workflow does not pass `--codes` + yet, so CI's Vulkan files are still round to nearest. - **Runners, per window** (2026-10-04): the export peak grows with the window squared (the same per-layer causal masks as XNNPACK), so one fixed 8vcpu runner could not carry any 32k window of Qwen3, Qwen2.5 or Llama-3.2-3B. The `targets` job runs `export-matrix`, which @@ -406,6 +417,32 @@ G5, Dimensity 9300+, Exynos 2500; GSM8K, RetrievalQA, IFEval, PopQA, BFCL, Fresh ## Known limits +- **Phi-4-mini** (2026-11-09): + - **Recipe.** It is export_llm's `phi_4_mini` with execupack's params and converter + (fused qkv/gate_up split), and matches transformers' fp32 Phi3ForCausalLM to 0.003 in + the logits on the gate's prompts. + - **LongRoPE.** It is static per file: ExecuTorch picks the long factors when the RoPE + table is longer than 4096. `ffn_dim_multiplier` is dropped from Phi's params, because + upstream's 1 doubles the table and would put a 4k file on the long factors. So 8k and + larger windows run the long factors from position 0, as llama.cpp does by context + size. HF uses them only past 4096 tokens, so short prompts there differ from HF (KL + 2.1, top token 14 of 16, more tool-eager). + - **Quantization.** Its fp32 tool-call decisions are near ties (0.40 to 0.78 on the + graded rows). GPTQ int4 kept 7 of 8. The int8 output head (`XnnpackPlan.output_bits`, + +0.27 GB) raised the mean but not the count on the .pte. + - **Override.** The published files went out through `verify --accept-gate REASON`, + which keeps the measured verdict (`passed: false`) beside the operator's reason. The + README states the shortfall per window. +- **The decision gate cannot see a thinking-only model's decision.** + Qwen3-4B-Thinking-2507's template opens `` whatever `enable_thinking` says. The + gate closes it empty, as the app does with thinking off, but this model reasons anyway: + fp32's first token is "Okay" on every row, so `measures_tool_calling` is false. A + generation through the reasoning on the published CPU 4k file reached a tool call on 6 of + 12 search rows within 600 tokens (the rest were still reasoning) and on none of the 4 + quiet ones. +- **32k exports of 4B-class models need about 143 GB with headroom**: more than an + r7i.4xlarge (123 GiB), within an r7i.8xlarge. + - Export memory grows with the square of the window: every attention layer of ExecuTorch 1.4.0's transformer builds its own window × window causal mask (not stored in the `.pte`). Qwen3-0.6B at 32k needs 30,064,771,072 bytes of masks alone and killed a From 500979b4121cebe6b39be259de50bcddd4f299d4 Mon Sep 17 00:00:00 2001 From: Codesmith Date: Mon, 9 Nov 2026 04:31:47 +0000 Subject: [PATCH 09/10] Tests: stub transformers in the accepted-gate test, which CI does not install --- tests/test_verify.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_verify.py b/tests/test_verify.py index c5786a1..09a478b 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -2,6 +2,7 @@ import hashlib import json +import sys import pytest @@ -80,9 +81,10 @@ def test_an_accepted_gate_keeps_its_measured_verdict_and_records_the_reason(tmp_ "export_mean": 0.51, }, ) - import transformers - - monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: None) + # verify imports AutoTokenizer only to hand it to gate.measure, stubbed above; CI has no transformers. + tokenizer = type("AutoTokenizer", (), {"from_pretrained": staticmethod(lambda *a, **k: None)}) + monkeypatch.setitem(sys.modules, "transformers", type(sys)("transformers")) + monkeypatch.setattr(sys.modules["transformers"], "AutoTokenizer", tokenizer, raising=False) with pytest.raises(verify.ExportError, match="decision gate failed"): verify.run(out, "xnnpack", tmp_path / "w1", gate_reference=reference) summary = verify.run(out, "xnnpack", tmp_path / "w2", gate_reference=reference, accept_gate="near a tie") From 73baf335829f8bcd4ac1c13a106ffc28ae34d6fe Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Mon, 9 Nov 2026 12:36:51 +0800 Subject: [PATCH 10/10] Publish only files the report lists, from the root and on releases too; tie a verdict to its gate reference Codex review of the whole PR: - publish-release had no check for model files its report does not list, and publish-hf did not check the output root. Both now share require_vouched: a .pte or .bin about to be uploaded must be in the report. - A verdict records the sha256 of the gate.json it was measured against. verify measures again when the reference it is given is a different one, instead of answering with the old verdict. Co-Authored-By: Claude Opus 5.5 --- pipeline/publish.py | 18 ++++++++++++------ pipeline/verify.py | 9 ++++++++- tests/test_publish.py | 24 ++++++++++++++++++++++++ tests/test_verify.py | 12 +++++++++++- 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/pipeline/publish.py b/pipeline/publish.py index 0d32cda..7301adb 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -74,6 +74,16 @@ def require_same_bytes(out_dir: Path, report: dict) -> None: raise ValueError(f"{entry['path']} is not the file its report describes (sha256 differs)") +def require_vouched(report: dict, paths: list[str]) -> None: + """Every model file about to be uploaded is one the report lists: an extra or renamed + .pte or .bin, in the backend folder or at the repo root, has no verdict and no digest + behind it (codex review, rounds five and six).""" + listed = {entry["path"] for entry in report["files"]} + unvouched = sorted(path for path in paths if path.endswith((".pte", ".bin")) and path not in listed) + if unvouched: + raise ValueError(f"{unvouched} would be uploaded but are not in the report; nothing vouches for them") + + def require_publishable(report: dict) -> None: """Refuse a report that has not passed every check its recipe calls for. @@ -138,12 +148,7 @@ def publish_hf( # A tokenizer inside any backend folder makes the app stop lending the root one to # every other folder (HuggingFaceClient.tokenizerFor), breaking the other backends. raise ValueError(f"tokenizer files must stay at the repo root, found {scoped}") - # Only what the report vouches for: a model file in the folder that the report does not - # list (an extra or renamed .pte) has no verdict and no digest (codex review, round five). - listed = {entry["path"] for entry in report["files"]} - unvouched = [path for path in new_paths if path.endswith((".pte", ".bin")) and path not in listed] - if unvouched: - raise ValueError(f"{unvouched} are in {folder} but not in its report; nothing vouches for them") + require_vouched(report, [*new_paths, *root_files]) require_same_bytes(out_dir, report) hf = hub.api() @@ -245,6 +250,7 @@ def publish_release( label = naming.window_label(report["window"]["context"]) hf_url = f"https://huggingface.co/{report['output_repo']}" candidates = [p for p in sorted(folder.iterdir()) if window_of(p.name) in (None, label)] + require_vouched(report, [str(p.relative_to(out_dir)) for p in candidates]) candidates.append(out_dir / report["tokenizer"]) assets = [str(p) for p in candidates if p.is_file() and p.stat().st_size < RELEASE_ASSET_LIMIT] notes = release_notes(report, hf_url) diff --git a/pipeline/verify.py b/pipeline/verify.py index 395f892..7ffbcdb 100644 --- a/pipeline/verify.py +++ b/pipeline/verify.py @@ -50,7 +50,13 @@ def run( # leaves a smoke result behind, and skipping on that alone let a second run report # success with nothing gated (codex review, 2026-11-09). recorded = report.get("gate") or {} - gated = gate_reference is None or recorded.get("passed") or (accept_gate and recorded.get("override")) + # A verdict counts only against the reference that produced it: a corrected gate.json + # is measured again, not answered with the old verdict (codex review, round six). + reference_sha = exporting.sha256(gate_reference) if gate_reference is not None else None + same_reference = recorded.get("reference_sha256") == reference_sha + gated = gate_reference is None or ( + same_reference and (recorded.get("passed") or (accept_gate and recorded.get("override"))) + ) # And only while the file is the one that was verified: a rebuilt .pte is a new file. listed = out_dir / report["files"][0]["path"] same = listed.is_file() and exporting.sha256(listed) == report["files"][0].get("sha256") @@ -117,6 +123,7 @@ def run( if not verdict["passed"] and accept_gate: verdict["override"] = {"reason": accept_gate} print(f" gate failed and accepted by the operator: {accept_gate}") + verdict["reference_sha256"] = exporting.sha256(gate_reference) report["gate"] = verdict gates.append(verdict) diff --git a/tests/test_publish.py b/tests/test_publish.py index fed1404..269bf12 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -306,3 +306,27 @@ def test_a_file_that_is_not_the_one_its_report_describes_is_never_published(tmp_ monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) with pytest.raises(ValueError, match="not the file its report describes"): publish.publish_hf(out, "xnnpack") + + +def test_a_model_file_the_report_does_not_list_is_never_uploaded_from_the_root(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "stray.pte").write_bytes(b"not verified") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"pte") + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(report(2048))) + monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) + with pytest.raises(ValueError, match="stray.pte.*not in the report"): + publish.publish_hf(out, "xnnpack") + + +def test_a_release_never_attaches_a_model_file_its_report_does_not_list(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"pte") + (out / "xnnpack" / "renamed.pte").write_bytes(b"not verified") + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(report(2048))) + monkeypatch.setattr(publish.subprocess, "run", lambda *a, **k: pytest.fail("reached gh")) + with pytest.raises(ValueError, match="renamed.pte.*not in the report"): + publish.publish_release(out, "xnnpack") diff --git a/tests/test_verify.py b/tests/test_verify.py index 09a478b..25669d9 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -46,9 +46,10 @@ def test_a_smoke_tested_report_is_gated_again_when_its_gate_is_missing_or_failed def test_a_report_that_passed_both_is_left_alone(tmp_path, reaches_smoke): - out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True}}) reference = tmp_path / "gate.json" reference.write_text("{}") + measured_against = hashlib.sha256(b"{}").hexdigest() + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True, "reference_sha256": measured_against}}) assert verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference)["passed"] out_ungated = write(tmp_path / "b", {"smoke": {"passed": True}}) assert verify.run(out_ungated, "xnnpack", tmp_path / "work")["passed"] @@ -98,3 +99,12 @@ def test_a_verified_report_is_verified_again_when_its_file_was_rebuilt(tmp_path, (out / "xnnpack" / "x-2k.pte").write_bytes(b"rebuilt") with pytest.raises(Ran): verify.run(out, "xnnpack", tmp_path / "work") + + +def test_a_passing_verdict_against_another_reference_is_measured_again(tmp_path, reaches_smoke): + reference = tmp_path / "gate.json" + reference.write_text('{"corrected": true}') + old = hashlib.sha256(b"{}").hexdigest() + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True, "reference_sha256": old}}) + with pytest.raises(Ran): + verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference)