diff --git a/docs/PLAN.md b/docs/PLAN.md index 9afeb07..138a9ae 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -131,6 +131,17 @@ so the app's `CompiledBackend.of` reads "vulkan". The Linux wheel's runner has n kernels and the runner no GPU, so the check is structural (`smoke.structural_check`: `forward` delegates to `VulkanBackend`); quality and speed are measured on the devices. +- **GPTQ codes, gated through the XNNPACK twin** (2026-11-09): `export-vulkan --codes + --gate-proxy .json>` puts the solve's codes into the + same torchao linears, as for XNNPACK. With no Vulkan runtime on the host, the XNNPACK file + built from the same codes is the gate. `export_xnnpack.check_gate_proxy` accepts it only + when it was built the same way: source repo and revision, window, codes sha256, + ExecuTorch version, and every recipe field that shapes the graph or the weights. Its own + gate must also have passed or been accepted. The proxy is recorded in the Vulkan report, + and `publish` refuses a GPTQ Vulkan file without one. Why: round to nearest kept none of + Phi-4-mini's 8 graded tool calls (mean 0.05 against fp32's 0.66, measured on an XNNPACK + file with the same quantization). The `export-vulkan` workflow does not pass `--codes` + yet, so CI's Vulkan files are still round to nearest. - **Runners, per window** (2026-10-04): the export peak grows with the window squared (the same per-layer causal masks as XNNPACK), so one fixed 8vcpu runner could not carry any 32k window of Qwen3, Qwen2.5 or Llama-3.2-3B. The `targets` job runs `export-matrix`, which @@ -406,6 +417,32 @@ G5, Dimensity 9300+, Exynos 2500; GSM8K, RetrievalQA, IFEval, PopQA, BFCL, Fresh ## Known limits +- **Phi-4-mini** (2026-11-09): + - **Recipe.** It is export_llm's `phi_4_mini` with execupack's params and converter + (fused qkv/gate_up split), and matches transformers' fp32 Phi3ForCausalLM to 0.003 in + the logits on the gate's prompts. + - **LongRoPE.** It is static per file: ExecuTorch picks the long factors when the RoPE + table is longer than 4096. `ffn_dim_multiplier` is dropped from Phi's params, because + upstream's 1 doubles the table and would put a 4k file on the long factors. So 8k and + larger windows run the long factors from position 0, as llama.cpp does by context + size. HF uses them only past 4096 tokens, so short prompts there differ from HF (KL + 2.1, top token 14 of 16, more tool-eager). + - **Quantization.** Its fp32 tool-call decisions are near ties (0.40 to 0.78 on the + graded rows). GPTQ int4 kept 7 of 8. The int8 output head (`XnnpackPlan.output_bits`, + +0.27 GB) raised the mean but not the count on the .pte. + - **Override.** The published files went out through `verify --accept-gate REASON`, + which keeps the measured verdict (`passed: false`) beside the operator's reason. The + README states the shortfall per window. +- **The decision gate cannot see a thinking-only model's decision.** + Qwen3-4B-Thinking-2507's template opens `` whatever `enable_thinking` says. The + gate closes it empty, as the app does with thinking off, but this model reasons anyway: + fp32's first token is "Okay" on every row, so `measures_tool_calling` is false. A + generation through the reasoning on the published CPU 4k file reached a tool call on 6 of + 12 search rows within 600 tokens (the rest were still reasoning) and on none of the 4 + quiet ones. +- **32k exports of 4B-class models need about 143 GB with headroom**: more than an + r7i.4xlarge (123 GiB), within an r7i.8xlarge. + - Export memory grows with the square of the window: every attention layer of ExecuTorch 1.4.0's transformer builds its own window × window causal mask (not stored in the `.pte`). Qwen3-0.6B at 32k needs 30,064,771,072 bytes of masks alone and killed a diff --git a/pipeline/__main__.py b/pipeline/__main__.py index daed58a..142ad72 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -60,7 +60,9 @@ def _verify(args) -> int: try: gate = Path(args.gate) if args.gate else None - summary = verify_module.run(Path(args.out), args.backend, Path(args.work), gate_reference=gate) + summary = verify_module.run( + Path(args.out), args.backend, Path(args.work), gate_reference=gate, accept_gate=args.accept_gate + ) except ExportError as error: print(f"verify failed: {error}", file=sys.stderr) return 2 @@ -126,6 +128,8 @@ def _export_vulkan(args) -> int: Path(args.work), context=args.context, keep_work=args.keep_work, + codes=Path(args.codes) if args.codes else None, + gate_proxy=Path(args.gate_proxy) if args.gate_proxy else None, backend="vulkan", ) ) @@ -446,6 +450,12 @@ def main(argv: list[str] | None = None) -> int: default=None, help="gate.json from `solve`: the file must decide like the fp32 model before it publishes", ) + verify.add_argument( + "--accept-gate", + default=None, + metavar="REASON", + help="publish even if the gate fails, recording this reason beside the measured verdict", + ) verify.set_defaults(func=_verify) audit = commands.add_parser("audit", help="gate a file that is already published, without rebuilding it") @@ -480,6 +490,19 @@ def main(argv: list[str] | None = None) -> int: vulkan.add_argument( "--keep-work", action="store_true", help="keep the work dir after success (a failure always leaves it)" ) + # The Vulkan delegate runs the same torchao Int8DynamicActivationIntxWeight linears as + # XNNPACK, so the solve's codes go in the same way. There is no Vulkan runtime on the + # host to gate the file, so its gate is the XNNPACK file solved from the same codes. + vulkan.add_argument( + "--codes", + default=None, + help="codes.pt from `solve`; without it the int4 weights are rounded to nearest", + ) + vulkan.add_argument( + "--gate-proxy", + default=None, + help="with --codes: the XNNPACK export-report-.json from the same codes that passed its gate", + ) vulkan.set_defaults(func=_export_vulkan) qnn = commands.add_parser("export-qnn", help="compile a Qualcomm HTP .pte for one chip") diff --git a/pipeline/calibration.py b/pipeline/calibration.py index c2333c7..5422503 100644 --- a/pipeline/calibration.py +++ b/pipeline/calibration.py @@ -42,6 +42,7 @@ from __future__ import annotations +import json from pathlib import Path # Kept short on purpose: every prompt costs an fp32 generation pass on the runner, and the @@ -273,9 +274,27 @@ def render_app(tokenizer, user_turn: str, thinking: bool = True) -> str: ignore), so the next token is the answer or the tool call itself. Calibration leaves it at the template's default and renders exactly as before. """ - messages = [{"role": "system", "content": APP_SYSTEM}, *APP_PRIMER, {"role": "user", "content": user_turn}] + system = {"role": "system", "content": APP_SYSTEM} extra = {} if thinking else {"enable_thinking": False} - return tokenizer.apply_chat_template(messages, tools=APP_TOOLS, tokenize=False, add_generation_prompt=True, **extra) + if system_carries_tools(tokenizer): + system["tools"] = json.dumps(APP_TOOLS) + else: + extra["tools"] = APP_TOOLS + messages = [system, *APP_PRIMER, {"role": "user", "content": user_turn}] + return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, **extra) + + +def system_carries_tools(tokenizer) -> bool: + """Whether the template reads its tools from the system message rather than ``tools=``. + + Phi-4-mini's does: it prints a system message's ``tools`` field, a JSON string, between + ``<|tool|>`` markers, and never reads the ``tools`` argument, so passing the schema the + usual way renders a prompt with no tools in it at all (codex review, 2026-11-09). The + app sends exactly that field: ``json.dumps`` of the list, ``", "`` between tools + (Phi4Prompt.render, ToolDefinition.asToolJson). + """ + template = getattr(tokenizer, "chat_template", None) + return isinstance(template, str) and "'tools' in message" in template def app_head(tokenizer) -> str: diff --git a/pipeline/convert.py b/pipeline/convert.py index 9b165fb..cbc8119 100644 --- a/pipeline/convert.py +++ b/pipeline/convert.py @@ -95,6 +95,24 @@ # as B, C and x stacked on dim 0, and ExecuTorch's ShortConv reads them apart. _IN_PROJ = ("conv.in_proj.weight", ("conv.B_proj.weight", "conv.C_proj.weight", "conv.x_proj.weight")) SPLIT_THREE = {"lfm2": _IN_PROJ, "lfm3": _IN_PROJ} +# Phi-4-mini (Phi3ForCausalLM) fuses q, k and v into one projection and the gate and up +# projections into another; everything else has Llama's names. HF splits them in forward: +# q, k, v at n_heads * head_dim and n_kv_heads * head_dim rows (Phi3Attention), gate before +# up in halves (Phi3MLP, gate_up_states.chunk(2, dim=-1)), as ExecuTorch's own +# examples/models/phi_4_mini/convert_weights.py does. The placeholders below never reach the +# checkpoint: convert() replaces each with its parts. +_PHI4_LAYER_KEYS = { + "self_attn.qkv_proj.weight": "attention.wqkv.weight", + "self_attn.o_proj.weight": "attention.wo.weight", + "input_layernorm.weight": "attention_norm.weight", + "post_attention_layernorm.weight": "ffn_norm.weight", + "mlp.gate_up_proj.weight": "feed_forward.w13.weight", + "mlp.down_proj.weight": "feed_forward.w2.weight", +} +_PHI4_FUSED = { + "attention.wqkv.weight": ("attention.wq.weight", "attention.wk.weight", "attention.wv.weight"), + "feed_forward.w13.weight": ("feed_forward.w1.weight", "feed_forward.w3.weight"), +} # Buffers some checkpoints carry that the model recomputes. _IGNORED = re.compile(r"(rotary_emb\.inv_freq|\.masked_bias|\.attn\.bias)$") @@ -107,9 +125,11 @@ # common one; every layer is a short convolution or an attention block, never both. "lfm2": (_LFM2_LAYER_KEYS, False), "lfm3": (_LFM3_LAYER_KEYS, False), + # HF RoPE (use_hf_rope), so q and k stay in HF's layout. + "phi4": (_PHI4_LAYER_KEYS, False), } # Converters whose per-layer map stands alone instead of extending _LAYER_KEYS. -_STANDALONE = {"lfm2", "lfm3"} +_STANDALONE = {"lfm2", "lfm3", "phi4"} # A conv layer and an attention layer carry different tensors, so the per-layer count that # the other families check cannot apply; LFM2 is checked by total instead. _MIXED_LAYERS = {"lfm2", "lfm3"} @@ -128,6 +148,23 @@ def unpermute(weight, n_heads: int): ) +def _phi4_parts(fused: str, tensor, config: dict) -> tuple: + """A fused Phi-4-mini projection cut where HF cuts it, refusing any other shape.""" + rows, cols = tensor.shape + if fused == "feed_forward.w13.weight": + hidden = int(config["intermediate_size"]) + if rows != 2 * hidden: + raise ValueError(f"gate_up_proj has {rows} rows, expected 2 x intermediate_size {hidden}") + return tensor[:hidden], tensor[hidden:] + n_heads = int(config["num_attention_heads"]) + head_dim = int(config.get("head_dim") or int(config["hidden_size"]) // n_heads) + q = n_heads * head_dim + kv = int(config.get("num_key_value_heads") or n_heads) * head_dim + if rows != q + 2 * kv: + raise ValueError(f"qkv_proj has {rows} rows, expected {q} + 2 x {kv}") + return tensor[:q], tensor[q : q + kv], tensor[q + kv :] + + def _shards(model_dir: Path) -> list[Path]: index = model_dir / "model.safetensors.index.json" if index.exists(): @@ -183,6 +220,12 @@ def convert(model_dir: Path, output: Path, converter: str, config: dict) -> dict for name, part in zip(split[1], torch.chunk(tensor, 3, dim=0), strict=True): state[head + name] = part continue + fused = next((f for f in _PHI4_FUSED if target.endswith(f)), None) if converter == "phi4" else None + if fused: + head = target[: -len(fused)] + for name, part in zip(_PHI4_FUSED[fused], _phi4_parts(fused, tensor, config), strict=True): + state[head + name] = part + continue if permute and target.endswith("attention.wq.weight"): tensor = unpermute(tensor, n_heads) elif permute and target.endswith("attention.wk.weight"): @@ -210,7 +253,9 @@ def convert(model_dir: Path, output: Path, converter: str, config: dict) -> dict if not widths or min(widths) < 6: raise ValueError(f"a layer is missing tensors: widths {widths}") else: - expected_per_layer = len(_LAYER_KEYS) + len(CONVERTERS[converter][0]) + # Phi-4-mini's six HF tensors per layer become Llama's nine. + extra = 0 if converter == "phi4" else len(CONVERTERS[converter][0]) + expected_per_layer = len(_LAYER_KEYS) + extra if per_layer != n_layers * expected_per_layer: raise ValueError(f"expected {n_layers} layers x {expected_per_layer} tensors, found {per_layer}") diff --git a/pipeline/eligibility.py b/pipeline/eligibility.py index 487ab3c..e4704a8 100644 --- a/pipeline/eligibility.py +++ b/pipeline/eligibility.py @@ -22,6 +22,7 @@ "lfm2": {"lfm25"}, # Ahead of the app (naming.AHEAD_OF_APP): it has no LFM3 template yet. "lfm3": {"lfm3"}, + "phi4_mini": {"phi4mini"}, } # What a release of each family is called. The architecture class alone is not enough: a # later generation can keep the class (a text-only "Qwen3.8" on Qwen3ForCausalLM would pass @@ -34,6 +35,9 @@ "llama": re.compile(r"llama-?3\.2(?![.\d])|smollm2(?!\d)", re.IGNORECASE), "gemma3": re.compile(r"gemma-?3(?![.\dn])", re.IGNORECASE), "smollm3": re.compile(r"smollm3(?!\d)", re.IGNORECASE), + # Phi-4-mini only: Phi-3 and Phi-3.5 share the architecture class, and the app's template + # is Phi-4-mini's (Phi4Prompt). + "phi4_mini": re.compile(r"phi-?4-mini(?![-.]?\d)", re.IGNORECASE), } _INSTRUCT_TOKENS = {"instruct", "it", "chat"} # Families whose chat models ship with no suffix at all and whose base models carry "-Base". diff --git a/pipeline/export_with_codes.py b/pipeline/export_with_codes.py index e8cfa83..ffb36a1 100644 --- a/pipeline/export_with_codes.py +++ b/pipeline/export_with_codes.py @@ -63,6 +63,59 @@ def inject(model, codes: dict, log=print) -> tuple[int, list[str]]: return injected, left +def quantizer(upstream, codes: dict, output_int8: bool): + """``upstream`` (source_transformation.quantize.quantize) with the solved codes put in, + and for ``output_int8`` the output head at int8 per channel rather than int4.""" + import torch + + codes_given = bool(codes) # main() refuses an empty codes file before this + if output_int8: + # The head is not int4 here, so its int4 codes have nowhere to go. + codes = {k: v for k, v in codes.items() if k != "output"} + + def quantize(model, qmode, *rest, **kwargs): + if qmode != "8da4w": + return upstream(model, qmode, *rest, **kwargs) + head = None + if output_int8: + # Hidden from the int4 pass, which takes every nn.Linear, then given int8 weights + # per channel with the same dynamic int8 activations + # (Int8DynamicActivationIntxWeightConfig, as upstream's own 8da8w mode builds them). + head, model.output = model.output, torch.nn.Identity() + quantised = upstream(model, qmode, *rest, **kwargs) + if head is not None: + from torchao.quantization import Int8DynamicActivationIntxWeightConfig, quantize_ + from torchao.quantization.granularity import PerAxis + + with torch.no_grad(): + quantize_( + head, + Int8DynamicActivationIntxWeightConfig(weight_dtype=torch.int8, weight_granularity=PerAxis(0)), + ) + quantised.output = head + print("output head quantized to int8 per channel") + if not codes_given: + return quantised + injected, left = inject(quantised, codes) + # And every int4 tensor must have its code: a partial codes file would otherwise + # export, be labelled GPTQ, and carry the rest rounded (codex review, 2026-11-09). The + # int8 head has no int4 code by design. + rounded = [n for n in left if not (output_int8 and n == "output")] + if rounded: + raise ValueError(f"no solved codes for {len(rounded)} int4 linears: {rounded[:5]}") + # Every code must find its tensor. If the solve was run against a different + # checkpoint, or a name moved, the export would otherwise finish, be labelled GPTQ, + # and carry weights that were quietly rounded instead. + if injected != len(codes): + missing = sorted(set(codes) - {n for n, _ in quantised.named_modules()}) + raise ValueError( + f"{len(codes)} solved tensors but {injected} were injected; these codes match no module: {missing[:5]}" + ) + return quantised + + return quantize + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True, help="the export_llm YAML the exporter wrote") @@ -72,6 +125,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="clear the short convolution's state at position zero (pipeline/lfm2_state.py)", ) + parser.add_argument( + "--output-int8", + action="store_true", + help="quantize the output head to int8 per channel instead of int4 (XnnpackPlan.output_bits)", + ) parser.add_argument( "--lfm3", action="store_true", @@ -93,31 +151,16 @@ def main(argv: list[str] | None = None) -> int: lfm3_model.apply() print("applied the LFM3 model definition (pipeline/lfm3_model.py)") - if args.codes: + if args.codes or args.output_int8: from executorch.examples.models.llama.source_transformation import quantize as quantize_module # weights_only: the codes file is an artifact passed between CI jobs, and it holds # nothing but tensors, so there is no reason to let it unpickle arbitrary objects. - codes = torch.load(args.codes, map_location="cpu", weights_only=True) - upstream = quantize_module.quantize - - def quantize(model, qmode, *rest, **kwargs): - quantised = upstream(model, qmode, *rest, **kwargs) - if qmode != "8da4w": - return quantised - injected, _ = inject(quantised, codes) - # Every code must find its tensor. If the solve was run against a different - # checkpoint, or a name moved, the export would otherwise finish, be labelled - # GPTQ, and carry weights that were quietly rounded instead. - if injected != len(codes): - missing = sorted(set(codes) - {n for n, _ in quantised.named_modules()}) - raise ValueError( - f"{len(codes)} solved tensors but {injected} were injected; " - f"these codes match no module: {missing[:5]}" - ) - return quantised - - quantize_module.quantize = quantize + codes = torch.load(args.codes, map_location="cpu", weights_only=True) if args.codes else {} + if args.codes and not codes: + # An empty codes file would export every linear rounded under a GPTQ label. + raise ValueError(f"{args.codes} holds no codes") + quantize_module.quantize = quantizer(quantize_module.quantize, codes, args.output_int8) from executorch.extension.llm.export.export_llm import main as export_llm_main diff --git a/pipeline/export_xnnpack.py b/pipeline/export_xnnpack.py index 286f2af..71d66ee 100644 --- a/pipeline/export_xnnpack.py +++ b/pipeline/export_xnnpack.py @@ -138,8 +138,63 @@ def pick_runner( return None +def check_gate_proxy(path: Path | None, sha: str, window: int, codes_sha256: str, expected: dict) -> dict: + """What stands in for a Vulkan file's gate: the XNNPACK report built from the same codes. + + The two delegates run the same torchao linears with the same codes, so the XNNPACK file + is the closest measurement of the Vulkan one the host can make. It counts only when it + was built the same way: the same source repo and revision, window (LongRoPE picks its + factors by window), codes file, toolchain and every recipe field that shapes the weights + or the graph (``expected``: model class, converter, params, qmode, group size, embedding + quantization, prefill chunk, output head). And only when its own gate passed, or was + accepted by an operator with a recorded reason. Anything else would label an ungated file + as gated (codex review, 2026-11-09). + """ + if path is None: + raise ExportError("a Vulkan export with solved codes needs --gate-proxy, the XNNPACK report that gates it") + try: + report = json.loads(Path(path).read_text()) + recipe, verdict = report.get("recipe") or {}, report.get("gate") or {} + problems = [] + if report.get("backend") != "xnnpack": + problems.append(f"it is a {report.get('backend')} report") + if (report.get("source") or {}).get("id") != expected["source_id"]: + problems.append("it was built from another source repo") + if (report.get("source") or {}).get("sha") != sha: + problems.append("it was built from another source revision") + if (report.get("window") or {}).get("context") != window: + problems.append(f"its window is {(report.get('window') or {}).get('context')}, not {window}") + if recipe.get("codes_sha256") != codes_sha256: + problems.append("it was built from other codes") + if (report.get("toolchain") or {}).get("executorch") != expected["executorch"]: + problems.append("it was built with another ExecuTorch") + for key, value in expected["recipe"].items(): + if recipe.get(key) != value: + problems.append(f"its {key} differs") + if not verdict.get("passed") and not (verdict.get("override") or {}).get("reason"): + problems.append("its decision gate did not pass" if verdict else "it has no decision gate") + if problems: + raise ExportError(f"gate proxy {path} does not stand for this export: {'; '.join(problems)}") + return { + "report": Path(path).name, + "file": report["files"][0]["path"], + "sha256": report["files"][0]["sha256"], + "gate": {k: verdict.get(k) for k in ("passed", "graded", "agreed", "fp32_mean", "export_mean", "override")}, + } + except ExportError: + raise + except (OSError, ValueError, KeyError, IndexError, TypeError) as error: + raise ExportError(f"gate proxy {path} could not be read: {error!r}") from error + + def recipe_fields( - recipe, qmode_label: str, solved: bool, backend: str, prefill_chunk: int, executorch: str | None = None + recipe, + qmode_label: str, + solved: bool, + backend: str, + prefill_chunk: int, + executorch: str | None = None, + output_bits: int = 4, ) -> dict: """The report's description of how the weights are stored, true to the recipe built. @@ -169,18 +224,22 @@ def recipe_fields( f"{prefill_chunk}, fp32 KV cache." ), } + head = output_bits == 8 return { **common, "group_size": recipe.group_size, "int4_codes": "gptq" if solved else "round-to-nearest", + **({"output_head": "int8 per channel"} if head else {}), "label": ( f"{qmode_label}-g{recipe.group_size}, int8 embeddings" + (", int4 codes solved with GPTQ" if solved else ", int4 codes rounded to nearest") + + (", int8 output head" if head else "") ), "description": ( f"ExecuTorch {version} `export_llm`: 8-bit dynamic activations " - f"and 4-bit weights in groups of {recipe.group_size}, int8 per-channel embeddings, " - f"{delegate}, prefill chunk {prefill_chunk}, fp32 KV cache." + f"and 4-bit weights in groups of {recipe.group_size}" + + (" (the output head 8-bit per channel)" if head else "") + + f", int8 per-channel embeddings, {delegate}, prefill chunk {prefill_chunk}, fp32 KV cache." ), } @@ -197,7 +256,11 @@ def run( codes: Path | None = None, qmode: str | None = None, allow_base: bool = False, + gate_proxy: Path | None = None, ) -> dict: + """``gate_proxy``: for a Vulkan export with solved codes, the XNNPACK export report that + gates it, since no Vulkan runtime runs on the host. It must have passed its decision gate + on the same source revision, window, codes and output head (check_gate_proxy).""" if backend not in BACKEND_CONFIG: raise ExportError(f"unknown backend {backend!r}; this exporter builds {sorted(BACKEND_CONFIG)}") cfg = settings.load() @@ -223,7 +286,14 @@ def run( raise ExportError("solved codes are int4; a GPTQ run exports 8da4w") recipe = dataclasses.replace(recipe, qmode=qmode) choice = sizing.choose_context( - arch, cfg.context_tiers, cfg.device_budget_bytes, cfg.runtime_overhead_bytes, budget, backend, recipe.qmode + arch, + cfg.context_tiers, + cfg.device_budget_bytes, + cfg.runtime_overhead_bytes, + budget, + backend, + recipe.qmode, + plan.output_bits if recipe.qmode == "8da4w" else 4, ) # Every tier is exported, one job per window; the app and the benchmarker decide what # fits a phone (the estimate goes into the report as fits_phone_budget). Only the host @@ -246,7 +316,39 @@ def run( f"a {context}-token export needs about {need:,} B with headroom (causal masks alone " f"{sizing.causal_mask_bytes(arch, context):,} B); this host has {budget:,} B" ) - resident = sizing.device_resident_bytes(arch, window, cfg.runtime_overhead_bytes, recipe.qmode) + output_bits = plan.output_bits if recipe.qmode == "8da4w" else 4 + resident = sizing.device_resident_bytes(arch, window, cfg.runtime_overhead_bytes, recipe.qmode, output_bits) + codes_sha256 = None + if codes is not None: + # The export reads the codes from a copy of its own, hashed here: a shared codes.pt + # replaced between the hash and the read would otherwise leave the report naming + # codes the file was not built from (codex review, 2026-11-09). + work_dir.mkdir(parents=True, exist_ok=True) + # Absolute: the export's subprocess runs inside work_dir (codex review, round five). + private = (work_dir / "codes.pt").resolve() + shutil.copyfile(codes, private) + codes, codes_sha256 = private, sha256(private) + proxy = None + if backend == "vulkan" and codes is not None: + # Checked before the export, which takes the better part of an hour at 32k. + expected = { + "source_id": source.id, + "executorch": toolchain()["executorch"], + "recipe": { + "model_class": plan.model_class, + "converter": plan.converter, + "params": plan.params, + "export_llm_qmode": QMODES[recipe.qmode], + "embedding_hqq": recipe.embedding_hqq, + "embedding_quantize": recipe.embedding_quantize, + "group_size": recipe.group_size, + "prefill_chunk": min(cfg.prefill_chunk, window), + "output_head": "int8 per channel" if output_bits == 8 else None, + }, + } + proxy = check_gate_proxy(gate_proxy, source.sha, window, codes_sha256, expected) + elif gate_proxy is not None: + raise ExportError("a gate proxy stands in for the gate of a Vulkan export with solved codes, and only that") output_repo = naming.output_repo(model_id, cfg.hub_org, cfg.repo_suffix) # The recipe goes in the file name, and a solve is a different recipe from rounding: a @@ -290,7 +392,8 @@ def run( # LFM3's model is this repository's (pipeline/lfm3_model.py), so its export always goes # through the wrapper too; its convolution clears its own state at position zero. state_fix = plan.converter == "lfm2" - if codes is not None or state_fix or plan.lfm3: + output_int8 = plan.output_bits == 8 and recipe.qmode == "8da4w" + if codes is not None or state_fix or plan.lfm3 or output_int8: command = [sys.executable, "-m", "pipeline.export_with_codes", "--config", str(config_path)] if codes is not None: command += ["--codes", str(codes)] @@ -298,6 +401,8 @@ def run( command += ["--lfm2-state-fix"] if plan.lfm3: command += ["--lfm3"] + if output_int8: + command += ["--output-int8"] print( f"==> export_llm (codes: {'gptq' if codes else 'round to nearest'}, state fix: {state_fix}, " f"lfm3: {plan.lfm3})" @@ -364,7 +469,15 @@ def run( "params": plan.params, "export_llm_qmode": QMODES[recipe.qmode], "embedding_hqq": recipe.embedding_hqq, - **recipe_fields(recipe, qmode_label, codes is not None, backend, min(cfg.prefill_chunk, window)), + **recipe_fields( + recipe, + qmode_label, + codes is not None, + backend, + min(cfg.prefill_chunk, window), + output_bits=output_bits, + ), + **({"codes_sha256": codes_sha256} if codes_sha256 else {}), }, "window": { "context": window, @@ -381,7 +494,7 @@ def run( {"path": pte_path_in_repo, "bytes": pte.stat().st_size, "sha256": sha256(pte)}, ], "estimates": { - "pte_bytes_estimate": sizing.pte_bytes_estimate(arch, window, recipe.qmode), + "pte_bytes_estimate": sizing.pte_bytes_estimate(arch, window, recipe.qmode, output_bits), "pte_bytes_actual": pte.stat().st_size, "export_peak_bytes_estimate": sizing.export_peak_bytes(arch, window), }, @@ -396,6 +509,7 @@ def run( "conversion": conversion, "metadata": method_values, "smoke": smoke_result, + **({"gate_proxy": proxy} if proxy else {}), "run": manifest.run_info(), } (backend_dir / "config.json").write_text( diff --git a/pipeline/families.py b/pipeline/families.py index 7a90f89..cd9b666 100644 --- a/pipeline/families.py +++ b/pipeline/families.py @@ -71,6 +71,13 @@ def supports(self, backend: str) -> bool: _LFM3_NO_VULKAN = "the Vulkan delegate has no kernel for the short convolution (as for LFM2)" _LFM3_NO_QNN = "not in ExecuTorch 1.5.1's Qualcomm SUPPORTED_LLM_MODELS registry" _LFM3_NO_MTK = "no LFM3 definition in ExecuTorch 1.5.1's examples/mediatek" +# Phi-4-mini has upstream paths on both NPUs (Qualcomm's registry entry phi_4_mini, MediaTek's +# phi.py), neither of which this pipeline has wired or measured yet. +_PHI4_NO_QNN = "ExecuTorch 1.5.1's Qualcomm registry has phi_4_mini, but execupack has not wired or validated it" +_PHI4_NO_MTK = ( + "ExecuTorch 1.5.1's examples/mediatek has phi.py (model_type phi4, where HF says phi3), " + "but execupack has not validated its LongRoPE" +) FAMILIES: tuple[Family, ...] = ( Family("qwen3", ("Qwen3ForCausalLM",)), @@ -96,6 +103,10 @@ def supports(self, backend: str) -> bool: ("Gemma3ForCausalLM",), {"xnnpack": _NOT_IN_EXPORT_LLM, "vulkan": _NOT_IN_EXPORT_LLM, "mtk": _MTK_GEMMA3}, ), + # Phi-4-mini: Llama's block with fused q/k/v and gate/up projections (pipeline/convert.py + # splits them), partial RoPE on 96 of 128 head dims, and LongRoPE. ExecuTorch 1.5.1 builds + # it from the same transformer (examples/models/phi_4_mini). + Family("phi4_mini", ("Phi3ForCausalLM",), {"qnn": _PHI4_NO_QNN, "mtk": _PHI4_NO_MTK}), Family( "smollm3", ("SmolLM3ForCausalLM",), @@ -356,6 +367,9 @@ class XnnpackPlan: model_class: str params: dict converter: str # key into pipeline.convert.CONVERTERS + # Bits for the output head's weights in an 8da4w export: 4 like every other linear, + # or 8 (per channel) where the int4 head is what costs the model its decisions. + output_bits: int = 4 @property def lfm3(self) -> bool: @@ -438,9 +452,73 @@ def xnnpack_plan(family: Family, config: dict) -> XnnpackPlan: params.update(use_scaled_rope=True, use_hf_rope=False, attention_qkv_bias=False) return XnnpackPlan("llama3_2", params, "llama") + if family.key == "phi4_mini": + return _phi4_mini_plan(c, params) + raise UnsupportedModel(f"no XNNPACK recipe for family {family.key}") +def _phi4_mini_plan(c: dict, params: dict) -> XnnpackPlan: + """Phi-4-mini's params, as ExecuTorch 1.5.1's examples/models/phi_4_mini/config/config.json + spells them, from the checkpoint's own config. + + LongRoPE (rope.hf_precompute_freqs_cis) divides the inverse frequencies by + ``short_factor`` or ``long_factor`` and scales cos and sin by ``attention_factor``, which + it derives from max over original position embeddings as HF does (1.19 here). HF picks + the factor per forward: short while the sequence is at most + ``original_max_position_embeddings`` (4096), long beyond. A precomputed table cannot, so + ExecuTorch picks once, by the table's length: long when it is over 4096. + + That length is the window -- unless ``ffn_dim_multiplier`` is set, in which case Rope + builds the table at twice the window (rope.py, "Sharded checkpoint"), and upstream's own + config sets it to 1. A 4k export would then run every prompt on the long factors, which + HF only uses past 4096 tokens. The multiplier does nothing else here (model_args reads it + only when hidden_dim is unset), so it is dropped: 2k and 4k windows use HF's short + factors exactly, and only windows past 4096 -- whose positions the short factors were + never trained for -- take the long ones, short prompts included. The gate grades that + against fp32 on short prompts, where HF uses the short factors. + """ + _require(c.get("model_type") == "phi3", f"model_type {c.get('model_type')!r} is not phi3") + _require(not c.get("attention_bias"), "Phi-4-mini with attention biases is not supported") + _require(not c.get("lm_head_bias"), "Phi-4-mini with an lm_head bias is not supported") + _require(bool(c.get("tie_word_embeddings")), "Phi-4-mini with untied embeddings is not validated") + partial = float(c.get("partial_rotary_factor") or 1.0) + _require(partial < 1.0, "full-width RoPE is Phi-3's, not Phi-4-mini's, and is not validated") + scaling = rope_scaling(c) + kind = (scaling or {}).get("rope_type") or (scaling or {}).get("type") + _require(kind == "longrope", f"RoPE scaling {kind!r}; the Phi-4-mini recipe implements longrope") + short, long = scaling.get("short_factor"), scaling.get("long_factor") + rotary_pairs = int(params["head_dim"] * partial) // 2 + _require( + isinstance(short, list) and isinstance(long, list) and len(short) == len(long) == rotary_pairs, + f"LongRoPE factors must be {rotary_pairs} long each", + ) + original = c.get("original_max_position_embeddings") or scaling.get("original_max_position_embeddings") + _require(isinstance(original, int) and original > 0, "LongRoPE without original_max_position_embeddings") + maximum = int(c["max_position_embeddings"]) + _require(maximum > original, "max_position_embeddings is not above the original context") + params.pop("ffn_dim_multiplier") + params.update( + use_scaled_rope=False, + use_hf_rope=True, + attention_qkv_bias=False, + partial_rotary_factor=partial, + original_max_position_embeddings=original, + max_position_embeddings=maximum, + rope_scaling_short_factor=[float(f) for f in short], + rope_scaling_long_factor=[float(f) for f in long], + ) + if scaling.get("attention_factor") is not None: + params["rope_scaling_attention_factor"] = float(scaling["attention_factor"]) + # The output head stays at 8 bits. Measured 2026-11-09 on the gate's prompts, eager, against + # fp32 (scratchpad expt.py): every linear int4 with GPTQ codes kept 7 of 8 tool calls at a + # mean of 0.457 against fp32's 0.663 -- the published recipe, and the .pte agreed (0.452) -- + # while the same with the head left in fp32 kept 8 of 8 at 0.521. Phi-4-mini's head is its + # tied 200,064 x 3072 embedding, a sixth of the model, and fp32 itself is unsure on these + # rows (0.40 to 0.78), so the head's rounding is enough to tip one. + return XnnpackPlan("phi_4_mini", params, "phi4", output_bits=8) + + def _lfm3_plan(c: dict) -> XnnpackPlan: """LFM3's params for pipeline/lfm3_model.py, from the fields of ExecuTorch's ModelArgs. diff --git a/pipeline/gate.py b/pipeline/gate.py index 87675a6..98602e6 100644 --- a/pipeline/gate.py +++ b/pipeline/gate.py @@ -94,17 +94,31 @@ def prompts(tokenizer) -> dict[str, str]: near tie that a 4-bit file broke the other way on two quiet rows without calling anything (2026-10-31). With the block closed in the prompt, the next token is the answer or the tool call, for every family; templates without the switch render as before. + + A reasoning-only template has no switch to read: Qwen3-4B-Thinking-2507's ends every + generation prompt in an open ```` whatever ``enable_thinking`` says, which would + put the first word of the reasoning back in the graded position. That block is closed + here the way Qwen3's own template closes it when the switch is off, and the way the app + sends a Qwen3 turn with thinking off (Qwen3Prompt.render, ``\\n\\n\\n\\n``). """ from pipeline import calibration def app(turn: str) -> str: - return calibration.render_app(tokenizer, turn, thinking=False) + return close_open_think(calibration.render_app(tokenizer, turn, thinking=False)) out = {q: app(f"{q}\n\n{calibration.TRAILER.format(subject=s)}") for q, s in SEARCH} out.update({q: app(q) for q in QUIET}) return out +def close_open_think(text: str) -> str: + """``text`` with a think block it leaves open at its very end closed empty.""" + head = text.rstrip("\n") + if head.endswith(""): + return head + "\n\n\n\n" + return text + + def reference(model, tokenizer) -> dict: """What the unquantised model would do: its top token per prompt, and how sure it is.""" import torch diff --git a/pipeline/manifest.py b/pipeline/manifest.py index cc78886..5935517 100644 --- a/pipeline/manifest.py +++ b/pipeline/manifest.py @@ -266,6 +266,21 @@ def readme(repo_id: str, reports: list[dict], hub_tags: list[str], license_files described.add(key) line = f"- {BACKEND_TITLES[r['backend']]} {target(r)}: {recipe['description']}" same = [x for x in reports if (x["backend"], x.get("target"), x["recipe"]["description"]) == key] + # Every window of this recipe that went out past its gate, each with its own numbers: + # one window's verdict cannot speak for the others (codex review, round five). + accepted = [] + for x in sorted(same, key=lambda x: x["window"]["context"]): + verdict = x.get("gate") or (x.get("gate_proxy") or {}).get("gate") or {} + reason = (verdict.get("override") or {}).get("reason") + if reason: + accepted.append((naming.window_label(x["window"]["context"]), verdict, reason)) + if accepted: + windows = ", ".join(f"{label} kept {v.get('agreed')} of {v.get('graded')}" for label, v, _ in accepted) + reasons = sorted({reason for _, _, reason in accepted}) + line += ( + f" Published past its decision gate, which asks that 90% of fp32's tool calls be kept: {windows}. " + f"Accepted because {' / '.join(reasons)}" + ) runs = sorted({x["run"]["url"] for x in same if x.get("run", {}).get("url")}) if runs: line += " Built by " + ", ".join(f"[run {i + 1}]({url})" for i, url in enumerate(runs)) + "." diff --git a/pipeline/naming.py b/pipeline/naming.py index e2ab9ae..9357533 100644 --- a/pipeline/naming.py +++ b/pipeline/naming.py @@ -128,8 +128,16 @@ def nominal_billions(name: str) -> float | None: return number / 1000 if hint.endswith("M") else number +# Releases whose own name carries no size. The app reads the size from the name +# (HubModel.parameterHint) and this pipeline refuses a name without one, so their outputs are +# named with it, appended: microsoft/Phi-4-mini-instruct is 3,836,021,760 parameters by the +# Hub's safetensors count and "3.8B" on its own model card. +SIZED_NAMES = {"microsoft/Phi-4-mini-instruct": "Phi-4-mini-instruct-3.8B"} + + def source_name(model_id: str) -> str: - return model_id.rsplit("/", 1)[-1] + """The name outputs are built from: the repo's own, with the size added where it has none.""" + return SIZED_NAMES.get(model_id) or model_id.rsplit("/", 1)[-1] def output_repo(model_id: str, hub_org: str, suffix: str) -> str: diff --git a/pipeline/publish.py b/pipeline/publish.py index 25c4642..7301adb 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -62,6 +62,61 @@ def superseded(path: str, folder: str, label: str) -> bool: return window_of(name) == label +def require_same_bytes(out_dir: Path, report: dict) -> None: + """Every file the report lists is, byte for byte, the file it describes: a verdict is + about the bytes it measured, and a .pte rebuilt or replaced after verify is not that file + (codex review, 2026-11-09).""" + for entry in report["files"]: + local = out_dir / entry["path"] + if not local.is_file(): + raise ValueError(f"{entry['path']} is listed in the report but not in {out_dir}") + if exporting.sha256(local) != entry.get("sha256"): + raise ValueError(f"{entry['path']} is not the file its report describes (sha256 differs)") + + +def require_vouched(report: dict, paths: list[str]) -> None: + """Every model file about to be uploaded is one the report lists: an extra or renamed + .pte or .bin, in the backend folder or at the repo root, has no verdict and no digest + behind it (codex review, rounds five and six).""" + listed = {entry["path"] for entry in report["files"]} + unvouched = sorted(path for path in paths if path.endswith((".pte", ".bin")) and path not in listed) + if unvouched: + raise ValueError(f"{unvouched} would be uploaded but are not in the report; nothing vouches for them") + + +def require_publishable(report: dict) -> None: + """Refuse a report that has not passed every check its recipe calls for. + + The workflow runs the smoke test in its own job and fails before reaching publish, but + publishing is also a command a person can run by hand, against a directory exported with + --skip-smoke. Refusing here is what makes "published" mean "ran and answered" rather than + "the workflow happened to be wired correctly that day". A report that was gated must have + passed it. A GPTQ file always has an fp32 reference to be gated against (the solve writes + it), so it must carry a passing gate of its own, or for Vulkan, which the host cannot run, + a gate proxy that passed: a GPTQ report with neither is one whose gate never ran or + crashed (codex review, 2026-11-09). Any other report with no gate (round-to-nearest + Vulkan) is published on its smoke test, as before. + """ + name = report["files"][0]["path"] + smoke = report.get("smoke") + if smoke is None: + raise ValueError( + f"{name} has no smoke result: run `python -m pipeline verify` on this directory before publishing it" + ) + if not smoke.get("passed"): + raise ValueError(f"{name} failed its smoke test: {'; '.join(smoke.get('problems', []))}") + verdict = report.get("gate") + if verdict is not None and not verdict.get("passed") and not (verdict.get("override") or {}).get("reason"): + raise ValueError(f"{name} failed its decision gate: {'; '.join(verdict.get('problems', []))}") + if (report.get("recipe") or {}).get("int4_codes") == "gptq": + if report.get("backend") == "vulkan": + proxy = (report.get("gate_proxy") or {}).get("gate") or {} + if not proxy.get("passed") and not (proxy.get("override") or {}).get("reason"): + raise ValueError(f"{name} has GPTQ codes and no passing gate proxy (the XNNPACK file from its codes)") + elif verdict is None: + raise ValueError(f"{name} has GPTQ codes and no decision gate: verify it with --gate before publishing") + + def publish_hf( out_dir: Path, backend: str, target: str | None = None, context: int | None = None, attempts: int = 6 ) -> str: @@ -78,18 +133,7 @@ def publish_hf( cfg = settings.load() report = load_report(out_dir, backend, target, context) - # The workflow runs the smoke test in its own job and fails before reaching this, but - # publish is also a command a person can run by hand, against a directory exported with - # --skip-smoke. Refusing here is what makes "published" mean "ran and answered" rather - # than "the workflow happened to be wired correctly that day". - smoke = report.get("smoke") - if smoke is None: - raise ValueError( - f"{report['files'][0]['path']} has no smoke result: run `python -m pipeline verify` " - "on this directory before publishing it" - ) - if not smoke.get("passed"): - raise ValueError(f"{report['files'][0]['path']} failed its smoke test: {'; '.join(smoke.get('problems', []))}") + require_publishable(report) repo_id = report["output_repo"] folder = _folder(backend, target) label = naming.window_label(report["window"]["context"]) @@ -104,6 +148,8 @@ def publish_hf( # A tokenizer inside any backend folder makes the app stop lending the root one to # every other folder (HuggingFaceClient.tokenizerFor), breaking the other backends. raise ValueError(f"tokenizer files must stay at the repo root, found {scoped}") + require_vouched(report, [*new_paths, *root_files]) + require_same_bytes(out_dir, report) hf = hub.api() hf.create_repo(repo_id, repo_type="model", exist_ok=True) @@ -197,11 +243,14 @@ def publish_release( ) -> str: """Create (or update) a GitHub release with this window's files under the 2 GiB limit.""" report = load_report(out_dir, backend, target, context) + require_publishable(report) + require_same_bytes(out_dir, report) folder = out_dir / _folder(backend, target) tag = release_tag(report) label = naming.window_label(report["window"]["context"]) hf_url = f"https://huggingface.co/{report['output_repo']}" candidates = [p for p in sorted(folder.iterdir()) if window_of(p.name) in (None, label)] + require_vouched(report, [str(p.relative_to(out_dir)) for p in candidates]) candidates.append(out_dir / report["tokenizer"]) assets = [str(p) for p in candidates if p.is_file() and p.stat().st_size < RELEASE_ASSET_LIMIT] notes = release_notes(report, hf_url) diff --git a/pipeline/sizing.py b/pipeline/sizing.py index 5cc7318..22c8f1e 100644 --- a/pipeline/sizing.py +++ b/pipeline/sizing.py @@ -113,17 +113,25 @@ def linear_bytes_per_param(qmode: str) -> float: raise ValueError(f"no size model for qmode {qmode!r}; known: {sorted(LINEAR_BYTES_PER_PARAM)}") from None -def pte_bytes_estimate(arch: Architecture, context: int, qmode: str = "8da4w") -> int: +def pte_bytes_estimate(arch: Architecture, context: int, qmode: str = "8da4w", output_bits: int = 4) -> int: + """``output_bits`` 8 prices the output head at one byte per weight (int8 per channel, + XnnpackPlan.output_bits) instead of the int4 rate every other linear is at.""" + per_param = linear_bytes_per_param(qmode) + head = arch.vocab_size * arch.dim + head_extra = head * (EMBEDDING_BYTES_PER_PARAM_INT8 - per_param) if qmode == "8da4w" and output_bits == 8 else 0 raw = ( arch.embedding_params * EMBEDDING_BYTES_PER_PARAM_INT8 - + arch.linear_params * linear_bytes_per_param(qmode) + + arch.linear_params * per_param + + head_extra + context * arch.head_dim * ROPE_TABLE_BYTES_PER_TOKEN_PER_HEAD_DIM ) return int(raw * PTE_OVERHEAD_FACTOR) -def device_resident_bytes(arch: Architecture, context: int, overhead: int, qmode: str = "8da4w") -> int: - return pte_bytes_estimate(arch, context, qmode) + kv_cache_bytes(arch, context) + overhead +def device_resident_bytes( + arch: Architecture, context: int, overhead: int, qmode: str = "8da4w", output_bits: int = 4 +) -> int: + return pte_bytes_estimate(arch, context, qmode, output_bits) + kv_cache_bytes(arch, context) + overhead def causal_mask_bytes(arch: Architecture, context: int) -> int: @@ -201,13 +209,14 @@ def choose_context( host_budget: int | None, backend: str = "xnnpack", qmode: str = "8da4w", + output_bits: int = 4, ) -> WindowChoice: """Largest tier whose phone residency and host export peak both fit. `qmode` sizes the linears: an fp32 file holds them at four bytes a weight, eight times an 8da4w one.""" rows = [] chosen = None for context in sorted(tiers, reverse=True): - resident = device_resident_bytes(arch, context, runtime_overhead, qmode) + resident = device_resident_bytes(arch, context, runtime_overhead, qmode, output_bits) peak = export_peak_bytes(arch, context) fits_device = resident <= device_budget fits_host = host_budget is None or host_need_bytes(arch, context, backend) <= host_budget diff --git a/pipeline/verify.py b/pipeline/verify.py index 02c0467..7ffbcdb 100644 --- a/pipeline/verify.py +++ b/pipeline/verify.py @@ -19,7 +19,7 @@ import json from pathlib import Path -from pipeline import families, gate, hub, settings, smoke +from pipeline import exporting, families, gate, hub, settings, smoke from pipeline.exporting import ExportError @@ -30,13 +30,37 @@ def reports(out_dir: Path, backend: str) -> list[Path]: return found -def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None = None) -> dict: - """Run the smoke test for every window in ``out_dir`` and write the results back.""" +def run( + out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None = None, accept_gate: str | None = None +) -> dict: + """Run the smoke test for every window in ``out_dir`` and write the results back. + + ``accept_gate`` is an operator's reason for publishing a file that fails its decision + gate. The verdict is kept as measured (``passed`` stays false) and the reason is recorded + beside it as ``override``; publish accepts a failed gate only with one. It exists for a + model whose fp32 decisions are themselves near ties, so that a 4-bit file cannot keep all + of them: Phi-4-mini, 7 of 8 kept, 2026-11-09. + """ results = [] gates: list[dict] = [] for path in reports(out_dir, backend): report = json.loads(path.read_text()) - if report.get("smoke") is not None: + # Verified already, unless a gate is asked for that this report has not passed: a + # first run that passed the smoke test and then failed (or never reached) the gate + # leaves a smoke result behind, and skipping on that alone let a second run report + # success with nothing gated (codex review, 2026-11-09). + recorded = report.get("gate") or {} + # A verdict counts only against the reference that produced it: a corrected gate.json + # is measured again, not answered with the old verdict (codex review, round six). + reference_sha = exporting.sha256(gate_reference) if gate_reference is not None else None + same_reference = recorded.get("reference_sha256") == reference_sha + gated = gate_reference is None or ( + same_reference and (recorded.get("passed") or (accept_gate and recorded.get("override"))) + ) + # And only while the file is the one that was verified: a rebuilt .pte is a new file. + listed = out_dir / report["files"][0]["path"] + same = listed.is_file() and exporting.sha256(listed) == report["files"][0].get("sha256") + if report.get("smoke") is not None and gated and same: print(f"==> {path.name}: already verified, leaving it") results.append(report["smoke"]) continue @@ -96,6 +120,10 @@ def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None "the quiet ones, so the gate checked that the export tracks its fp32 model " "and not that it still calls tools" ) + if not verdict["passed"] and accept_gate: + verdict["override"] = {"reason": accept_gate} + print(f" gate failed and accepted by the operator: {accept_gate}") + verdict["reference_sha256"] = exporting.sha256(gate_reference) report["gate"] = verdict gates.append(verdict) @@ -106,7 +134,7 @@ def run(out_dir: Path, backend: str, work_dir: Path, gate_reference: Path | None if failed: problems = "; ".join(p for r in failed for p in r.get("problems", [])) raise ExportError(f"smoke test failed: {problems}") - regressed = [g for g in gates if not g.get("passed")] + regressed = [g for g in gates if not g.get("passed") and not g.get("override")] if regressed: problems = "; ".join(p for g in regressed for p in g.get("problems", [])) raise ExportError(f"decision gate failed: {problems}") diff --git a/tests/conftest.py b/tests/conftest.py index 2a840e7..66691a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,7 @@ "HuggingFaceTB/SmolLM2-360M-Instruct": 361_821_120, "Qwen/Qwen3-30B-A3B": 30_532_122_624, "google/gemma-3-1b-it": 999_885_952, + "microsoft/Phi-4-mini-instruct": 3_836_021_760, } CONFIG_FILES = { "Qwen/Qwen3-0.6B": "qwen3-0.6b", @@ -33,6 +34,7 @@ "HuggingFaceTB/SmolLM2-360M-Instruct": "smollm2-360m-instruct", "Qwen/Qwen3-30B-A3B": "qwen3-30b-a3b", "google/gemma-3-1b-it": "gemma-3-1b-it", + "microsoft/Phi-4-mini-instruct": "phi-4-mini-instruct", } diff --git a/tests/fixtures/et-phi_4_mini.params.json b/tests/fixtures/et-phi_4_mini.params.json new file mode 100644 index 0000000..59f4440 --- /dev/null +++ b/tests/fixtures/et-phi_4_mini.params.json @@ -0,0 +1,19 @@ +{ + "dim": 3072, + "ffn_dim_multiplier": 1, + "hidden_dim": 8192, + "n_heads": 24, + "n_kv_heads": 8, + "n_layers": 32, + "norm_eps": 1e-05, + "rope_theta": 10000.0, + "use_scaled_rope": false, + "vocab_size": 200064, + "use_hf_rope": true, + "partial_rotary_factor": 0.75, + "attention_qkv_bias": false, + "original_max_position_embeddings": 4096, + "max_position_embeddings": 131072, + "rope_scaling_short_factor": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "rope_scaling_long_factor": [1.0, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77] +} diff --git a/tests/fixtures/phi-4-mini-instruct.config.json b/tests/fixtures/phi-4-mini-instruct.config.json new file mode 100644 index 0000000..679e70d --- /dev/null +++ b/tests/fixtures/phi-4-mini-instruct.config.json @@ -0,0 +1,144 @@ +{ + "_name_or_path": "Phi-4-mini-instruct", + "architectures": [ + "Phi3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_phi3.Phi3Config", + "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM", + "AutoTokenizer": "Xenova/gpt-4o" + }, + "bos_token_id": 199999, + "embd_pdrop": 0.0, + "eos_token_id": 199999, + "full_attn_mod": 1, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "interpolate_factor": 1, + "lm_head_bias": false, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "phi3", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "original_max_position_embeddings": 4096, + "pad_token_id": 199999, + "partial_rotary_factor": 0.75, + "resid_pdrop": 0.0, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "long_factor": [ + 1, + 1.118320672, + 1.250641126, + 1.398617824, + 1.564103225, + 1.74916897, + 1.956131817, + 2.187582649, + 2.446418898, + 2.735880826, + 3.059592084, + 3.421605075, + 3.826451687, + 4.279200023, + 4.785517845, + 5.351743533, + 5.984965424, + 6.693110555, + 7.485043894, + 8.370679318, + 9.36110372, + 10.4687158, + 11.70738129, + 13.09260651, + 14.64173252, + 16.37415215, + 18.31155283, + 20.47818807, + 22.90118105, + 25.61086418, + 28.64115884, + 32.03, + 32.1, + 32.13, + 32.23, + 32.6, + 32.61, + 32.64, + 32.66, + 32.7, + 32.71, + 32.93, + 32.97, + 33.28, + 33.49, + 33.5, + 44.16, + 47.77 + ], + "short_factor": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "type": "longrope" + }, + "rope_theta": 10000.0, + "sliding_window": 262144, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.45.0", + "use_cache": true, + "vocab_size": 200064 +} diff --git a/tests/test_convert.py b/tests/test_convert.py index 4f5cfd9..eb0e28e 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -157,3 +157,42 @@ def test_the_fused_convolution_projection_splits_into_three(): target, parts = convert.SPLIT_THREE["lfm2"] assert target == "conv.in_proj.weight" assert parts == ("conv.B_proj.weight", "conv.C_proj.weight", "conv.x_proj.weight") + + +def test_phi4_fused_projections_split_where_hf_splits_them(tmp_path): + config = {**tiny_config(), "hidden_size": HIDDEN, "intermediate_size": FFN, "head_dim": HEAD_DIM} + plain = tiny_checkpoint() + fused = {k: v for k, v in plain.items() if "_proj" not in k or "o_proj" in k or "down_proj" in k} + for i in range(2): + p = f"model.layers.{i}." + fused[p + "self_attn.qkv_proj.weight"] = torch.cat( + [plain[p + f"self_attn.{n}_proj.weight"] for n in "qkv"], dim=0 + ) + fused[p + "mlp.gate_up_proj.weight"] = torch.cat( + [plain[p + "mlp.gate_proj.weight"], plain[p + "mlp.up_proj.weight"]], dim=0 + ) + phi_dir = tmp_path / "phi" + phi_dir.mkdir() + safetensors_torch.save_file(fused, str(phi_dir / "model.safetensors")) + convert.convert(phi_dir, tmp_path / "phi.pth", "phi4", config) + got = torch.load(tmp_path / "phi.pth") + for i in range(2): + p, e = f"model.layers.{i}.", f"layers.{i}." + assert torch.equal(got[e + "attention.wq.weight"], plain[p + "self_attn.q_proj.weight"]) + assert torch.equal(got[e + "attention.wk.weight"], plain[p + "self_attn.k_proj.weight"]) + assert torch.equal(got[e + "attention.wv.weight"], plain[p + "self_attn.v_proj.weight"]) + assert torch.equal(got[e + "feed_forward.w1.weight"], plain[p + "mlp.gate_proj.weight"]) + assert torch.equal(got[e + "feed_forward.w3.weight"], plain[p + "mlp.up_proj.weight"]) + assert not any(k.endswith(("wqkv.weight", "w13.weight")) for k in got) + assert torch.equal(got["output.weight"], got["tok_embeddings.weight"]) + + +def test_phi4_fused_projection_of_the_wrong_height_is_refused(tmp_path): + config = {**tiny_config(layers=1), "hidden_size": HIDDEN, "intermediate_size": FFN + 2, "head_dim": HEAD_DIM} + sd = {k: v for k, v in tiny_checkpoint(layers=1).items() if "_proj" not in k or "o_proj" in k or "down_proj" in k} + sd["model.layers.0.self_attn.qkv_proj.weight"] = torch.zeros((HEADS + 2 * KV_HEADS) * HEAD_DIM, HIDDEN) + sd["model.layers.0.mlp.gate_up_proj.weight"] = torch.zeros(2 * FFN, HIDDEN) + (tmp_path / "m").mkdir() + safetensors_torch.save_file(sd, str(tmp_path / "m" / "model.safetensors")) + with pytest.raises(ValueError, match="gate_up_proj has"): + convert.convert(tmp_path / "m", tmp_path / "out.pth", "phi4", config) diff --git a/tests/test_eligibility.py b/tests/test_eligibility.py index ad77085..b5e31cb 100644 --- a/tests/test_eligibility.py +++ b/tests/test_eligibility.py @@ -156,3 +156,16 @@ def test_a_chat_model_named_without_a_suffix_is_still_published(): # The other half of the rule: Qwen3 and LFM2.5-2.6B ship chat models with no suffix, and # refusing "base" by name alone would drop them. assert evaluate("Qwen/Qwen3-1.7B").eligible + + +def test_phi4_mini_is_eligible_for_cpu_and_gpu_only(source): + verdict = eligibility.evaluate(source("microsoft/Phi-4-mini-instruct"), settings.load()) + assert verdict.reasons == [] + assert (verdict.family, verdict.variant) == ("phi4_mini", "instruct") + assert verdict.export_backends == ["xnnpack", "vulkan"] + + +def test_phi_3_5_mini_shares_the_class_and_is_still_refused(source): + config = {**hf_config("microsoft/Phi-4-mini-instruct"), "partial_rotary_factor": 1.0} + verdict = eligibility.evaluate(source("microsoft/Phi-3.5-mini-instruct", config=config), settings.load()) + assert verdict.export_backends == [] diff --git a/tests/test_export_xnnpack.py b/tests/test_export_xnnpack.py index 5c40756..ce8cd19 100644 --- a/tests/test_export_xnnpack.py +++ b/tests/test_export_xnnpack.py @@ -110,3 +110,82 @@ def test_an_fp32_report_does_not_claim_int4_weights(): gptq = export_xnnpack.recipe_fields(cfg.xnnpack, "8da4w-gptq", True, "xnnpack", 2048) assert rtn["int4_codes"] == "round-to-nearest" and gptq["int4_codes"] == "gptq" assert "4-bit weights in groups of" in rtn["description"] and rtn["group_size"] == cfg.xnnpack.group_size + + +RECIPE = { + "model_class": "phi_4_mini", + "converter": "phi4", + "params": {"dim": 3072}, + "export_llm_qmode": "8da4w", + "embedding_hqq": False, + "embedding_quantize": "8,0", + "group_size": 32, + "prefill_chunk": 2048, + "output_head": "int8 per channel", +} +EXPECTED = {"source_id": "microsoft/Phi-4-mini-instruct", "executorch": "1.5.1", "recipe": RECIPE} + + +def _proxy(tmp_path, **changes): + import json + + report = { + "backend": "xnnpack", + "source": {"id": "microsoft/Phi-4-mini-instruct", "sha": "a" * 40}, + "toolchain": {"executorch": "1.5.1"}, + "window": {"context": 8192}, + "recipe": {**RECIPE, "codes_sha256": "c0de"}, + "gate": {"passed": True, "graded": 8, "agreed": 8, "fp32_mean": 0.66, "export_mean": 0.53}, + "files": [{"path": "xnnpack/x-8k.pte", "sha256": "ff"}], + } + for key, value in changes.items(): + report[key] = value + path = tmp_path / "export-report-8k.json" + path.write_text(json.dumps(report)) + return path + + +def test_a_gate_proxy_counts_only_for_the_same_build(tmp_path): + import copy + + from pipeline import export_xnnpack + + ok = export_xnnpack.check_gate_proxy(_proxy(tmp_path), "a" * 40, 8192, "c0de", EXPECTED) + assert ok["gate"]["passed"] and ok["file"] == "xnnpack/x-8k.pte" + four_bit_head = copy.deepcopy(EXPECTED) + four_bit_head["recipe"]["output_head"] = None + other_repo = {**EXPECTED, "source_id": "microsoft/Phi-4-mini-reasoning"} + other_toolchain = {**EXPECTED, "executorch": "1.6.0"} + for changes, args, match in ( + ({}, ("b" * 40, 8192, "c0de", EXPECTED), "another source revision"), + ({}, ("a" * 40, 4096, "c0de", EXPECTED), "window is 8192"), + ({}, ("a" * 40, 8192, "beef", EXPECTED), "other codes"), + ({}, ("a" * 40, 8192, "c0de", four_bit_head), "output_head differs"), + ({}, ("a" * 40, 8192, "c0de", other_repo), "another source repo"), + ({}, ("a" * 40, 8192, "c0de", other_toolchain), "another ExecuTorch"), + ( + {"recipe": {**RECIPE, "codes_sha256": "c0de", "embedding_quantize": "4,32"}}, + ("a" * 40, 8192, "c0de", EXPECTED), + "embedding_quantize differs", + ), + ({"gate": {"passed": False}}, ("a" * 40, 8192, "c0de", EXPECTED), "did not pass"), + ({"backend": "vulkan"}, ("a" * 40, 8192, "c0de", EXPECTED), "vulkan report"), + ({"files": []}, ("a" * 40, 8192, "c0de", EXPECTED), "could not be read"), + ): + with pytest.raises(export_xnnpack.ExportError, match=match): + export_xnnpack.check_gate_proxy(_proxy(tmp_path, **changes), *args) + with pytest.raises(export_xnnpack.ExportError, match="needs --gate-proxy"): + export_xnnpack.check_gate_proxy(None, "a" * 40, 8192, "c0de", EXPECTED) + accepted = {"passed": False, "agreed": 7, "graded": 8, "override": {"reason": "near a tie"}} + assert export_xnnpack.check_gate_proxy(_proxy(tmp_path, gate=accepted), "a" * 40, 8192, "c0de", EXPECTED) + + +def test_an_int8_head_is_priced_at_a_byte_a_weight(source): + from pipeline import families, sizing + + arch = families.architecture(source("microsoft/Phi-4-mini-instruct").config, 3_836_021_760) + four, eight = sizing.pte_bytes_estimate(arch, 2048), sizing.pte_bytes_estimate(arch, 2048, output_bits=8) + head = 200_064 * 3072 + assert 0.4 * head < eight - four < 0.5 * head # (1 - 0.5625) bytes per weight, plus overhead + rows = sizing.choose_context(arch, (2048,), 5_000_000_000, 500_000_000, None, output_bits=8).table + assert rows[0]["device_resident_bytes"] == sizing.device_resident_bytes(arch, 2048, 500_000_000, output_bits=8) diff --git a/tests/test_families.py b/tests/test_families.py index eec040a..93c4a87 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -217,3 +217,55 @@ def test_one_attention_layer_per_chunk_is_only_for_hybrids(): # An explicit all-attention layout is still not a hybrid: no chunk per layer for it. qwen["layer_types"] = ["full_attention"] * int(qwen["num_hidden_layers"]) assert families.mtk_chunks_one_attention_each(qwen) is None + + +PHI4 = "microsoft/Phi-4-mini-instruct" + + +def test_phi4_mini_params_equal_executorchs_own_but_the_table_doubling_multiplier(): + # ExecuTorch's config sets ffn_dim_multiplier 1, which makes Rope build its table at twice + # the window and so run a 4k export on LongRoPE's long factors (families._phi4_mini_plan). + expected = expected_params("et-phi_4_mini.params.json") + expected.pop("ffn_dim_multiplier") + plan = plan_for(hf_config(PHI4)) + assert plan.params == expected + assert (plan.model_class, plan.converter) == ("phi_4_mini", "phi4") + + +def test_phi4_mini_shapes_outside_the_validated_one_are_refused(): + for change, match in ( + ({"partial_rotary_factor": 1.0}, "full-width RoPE"), + ({"tie_word_embeddings": False}, "untied"), + ({"rope_scaling": None}, "longrope"), + ({"attention_bias": True}, "attention biases"), + ): + config = copy.deepcopy(hf_config(PHI4)) + config.update(change) + with pytest.raises(families.UnsupportedModel, match=match): + plan_for(config) + config = copy.deepcopy(hf_config(PHI4)) + config["rope_scaling"]["long_factor"] = config["rope_scaling"]["long_factor"][:-1] + with pytest.raises(families.UnsupportedModel, match="48 long"): + plan_for(config) + + +@pytest.mark.parametrize("window, factor", [(2048, "short"), (4096, "short"), (8192, "long"), (32768, "long")]) +def test_phi4_mini_rope_table_takes_hf_short_factors_up_to_4096(window, factor): + rope = pytest.importorskip("executorch.examples.models.llama.rope") + model_args = pytest.importorskip("executorch.examples.models.llama.model_args") + torch = pytest.importorskip("torch") + params = plan_for(hf_config(PHI4)).params + built = rope.Rope(model_args.ModelArgs(**params, max_context_len=window, max_seq_len=min(window, 2048))) + assert built.freqs_cos.shape[0] == window + chosen = params[f"rope_scaling_{factor}_factor"] + want_cos, _ = rope.hf_precompute_freqs_cis( + params["head_dim"], + window, + params["rope_theta"], + params["partial_rotary_factor"], + short_factor=chosen, + long_factor=chosen, + original_max_pos=4096, + max_position_embeddings=131072, + ) + assert torch.equal(built.freqs_cos, want_cos) diff --git a/tests/test_gate.py b/tests/test_gate.py index 25dd268..f863f28 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -137,3 +137,64 @@ def test_measuring_gate_records_throughput_and_passes(): assert result["decode_tok_per_sec"] == 42.12 assert result["prompt_tokens"] == 16 assert result["generated_tokens"] == 32 + + +class _Template: + """A chat template reduced to its generation prompt, as three real families end it.""" + + def __init__(self, opener): + self.opener = opener + + def apply_chat_template(self, messages, tools=None, tokenize=False, add_generation_prompt=True, **extra): + body = "".join(f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n" for m in messages) + return body + "<|im_start|>assistant\n" + self.opener(extra.get("enable_thinking", True)) + + +def test_a_reasoning_only_template_has_its_think_block_closed_like_the_switch_closes_qwen3s(): + # Qwen3-4B-Thinking-2507 opens the block whatever enable_thinking says; hybrid Qwen3 + # closes it empty when the switch is off. The gate has to grade both on the decision. + thinking_only = gate.prompts(_Template(lambda thinking: "\n")) + hybrid = gate.prompts(_Template(lambda thinking: "" if thinking else "\n\n\n\n")) + assert thinking_only == hybrid + assert all(p.endswith("<|im_start|>assistant\n\n\n\n\n") for p in thinking_only.values()) + + +def test_a_template_without_a_think_block_is_graded_as_it_renders(): + plain = gate.prompts(_Template(lambda thinking: "")) + assert all(p.endswith("<|im_start|>assistant\n") for p in plain.values()) + assert gate.close_open_think("a \nreasoning\n\nanswer") == "a \nreasoning\n\nanswer" + + +# microsoft/Phi-4-mini-instruct's chat_template (tokenizer_config.json, rev cfbefacb), verbatim. +PHI4_TEMPLATE = ( + "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and " + "message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + " + "message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + " + "message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}" + "{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}" +) + + +class _Jinja: + """apply_chat_template reduced to rendering the template, as transformers does.""" + + def __init__(self, template): + import jinja2 + + self.chat_template = template + self._template = jinja2.Environment().from_string(template) + + def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True, **extra): + return self._template.render( + messages=messages, add_generation_prompt=add_generation_prompt, eos_token="", **extra + ) + + +def test_phi4_mini_gets_the_tool_schema_on_its_system_turn_as_the_app_sends_it(): + import json + + rendered = gate.prompts(_Jinja(PHI4_TEMPLATE))["What is Concord the capital of?"] + # Phi4Prompt.render: the system text, then the tools as one JSON array between markers. + head = f"<|system|>{calibration.APP_SYSTEM}<|tool|>{json.dumps(calibration.APP_TOOLS)}<|/tool|><|end|>" + assert rendered.startswith(head) + assert rendered.endswith("<|assistant|>") diff --git a/tests/test_naming.py b/tests/test_naming.py index db7d88d..176f0e2 100644 --- a/tests/test_naming.py +++ b/tests/test_naming.py @@ -105,3 +105,13 @@ def test_window_label(): assert naming.window_label(32768) == "32k" assert naming.window_label(8192) == "8k" assert naming.window_label(3000) == "3000" + + +def test_phi4_mini_is_named_with_its_size_and_reads_as_phi4mini_in_the_app(): + repo = naming.output_repo("microsoft/Phi-4-mini-instruct", "experimentalmachines", "-ExecuTorch") + assert repo == "experimentalmachines/Phi-4-mini-instruct-3.8B-ExecuTorch" + for backend in ("xnnpack", "vulkan"): + path = f"{backend}/" + naming.cpu_gpu_file("microsoft/Phi-4-mini-instruct", backend, "8da4w-gptq", 4096) + assert naming.check_app_rules(repo, path, backend) == [] + assert naming.app_family(naming.app_model_name(repo, path)) == "phi4mini" + assert naming.app_size_hint(repo.split("/")[1]) == "3.8B" diff --git a/tests/test_publish.py b/tests/test_publish.py index 9519e76..269bf12 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -1,5 +1,6 @@ """Publishing one window into a repo that already holds other windows and backends.""" +import hashlib import json from types import SimpleNamespace @@ -20,7 +21,14 @@ def report(context, backend="xnnpack", target=None, sha="abcdef0123456789", run= "toolchain": {"executorch": "1.4.0"}, "recipe": {"label": "8da4w-g32, int8 embeddings", "description": "recipe text."}, "window": {"context": context, "kv_cache_bytes_per_token": 229_376, "fits_phone_budget": context <= 8192}, - "files": [{"path": f"{folder}/Qwen3-1.7B-8da4w-{label}.pte", "bytes": 1_000 * context, "sha256": "ff"}], + # The tests write b"pte" as the file; publish checks the bytes against this. + "files": [ + { + "path": f"{folder}/Qwen3-1.7B-8da4w-{label}.pte", + "bytes": 1_000 * context, + "sha256": hashlib.sha256(b"pte").hexdigest(), + } + ], "metadata": {"get_max_context_len": context, "get_max_seq_len": min(2048, context)}, "smoke": {"passed": True, "answered": True}, "run": {"url": run}, @@ -246,3 +254,79 @@ def test_load_report_wants_exactly_one_report(tmp_path): with pytest.raises(ValueError, match="pass --context"): publish.load_report(tmp_path, "xnnpack") assert publish.load_report(tmp_path, "xnnpack", context=8192)["window"]["context"] == 8192 + + +def test_a_report_that_failed_its_decision_gate_is_never_published(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"pte") + failed = {**report(2048), "gate": {"passed": False, "problems": ["kept fp32's choice on 7 of 8 calling rows"]}} + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(failed)) + monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) + with pytest.raises(ValueError, match="failed its decision gate: kept fp32's choice on 7 of 8"): + publish.publish_hf(out, "xnnpack") + + +def test_a_gptq_file_needs_its_own_gate_or_for_vulkan_a_passing_proxy(): + gptq = {"int4_codes": "gptq", "label": "x", "description": "x"} + cpu = {**report(2048), "recipe": gptq} + with pytest.raises(ValueError, match="no decision gate"): + publish.require_publishable(cpu) + publish.require_publishable({**cpu, "gate": {"passed": True}}) + gpu = {**report(2048, backend="vulkan"), "recipe": gptq} + with pytest.raises(ValueError, match="no passing gate proxy"): + publish.require_publishable(gpu) + with pytest.raises(ValueError, match="no passing gate proxy"): + publish.require_publishable({**gpu, "gate_proxy": {"gate": {"passed": False}}}) + publish.require_publishable({**gpu, "gate_proxy": {"gate": {"passed": True}}}) + # Round to nearest has no fp32 reference to be gated against, as before. + publish.require_publishable({**report(2048, backend="vulkan"), "recipe": {"int4_codes": "round-to-nearest"}}) + + +def test_a_failed_gate_publishes_only_with_an_operators_recorded_reason(): + gptq = {"int4_codes": "gptq", "label": "x", "description": "x"} + failed = {"passed": False, "agreed": 7, "graded": 8, "problems": ["7 of 8"]} + cpu = {**report(2048), "recipe": gptq, "gate": failed} + with pytest.raises(ValueError, match="failed its decision gate"): + publish.require_publishable(cpu) + with pytest.raises(ValueError, match="failed its decision gate"): + publish.require_publishable({**cpu, "gate": {**failed, "override": {"reason": ""}}}) + publish.require_publishable({**cpu, "gate": {**failed, "override": {"reason": "fp32 is near a tie"}}}) + gpu = {**report(2048, backend="vulkan"), "recipe": gptq} + publish.require_publishable({**gpu, "gate_proxy": {"gate": {**failed, "override": {"reason": "near a tie"}}}}) + + +def test_a_file_that_is_not_the_one_its_report_describes_is_never_published(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"rebuilt after verify") + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(report(2048))) + monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) + with pytest.raises(ValueError, match="not the file its report describes"): + publish.publish_hf(out, "xnnpack") + + +def test_a_model_file_the_report_does_not_list_is_never_uploaded_from_the_root(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "stray.pte").write_bytes(b"not verified") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"pte") + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(report(2048))) + monkeypatch.setattr(publish.hub, "api", lambda: pytest.fail("reached the Hub")) + with pytest.raises(ValueError, match="stray.pte.*not in the report"): + publish.publish_hf(out, "xnnpack") + + +def test_a_release_never_attaches_a_model_file_its_report_does_not_list(tmp_path, monkeypatch): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "tokenizer.json").write_text("{}") + (out / "xnnpack" / "Qwen3-1.7B-8da4w-2k.pte").write_bytes(b"pte") + (out / "xnnpack" / "renamed.pte").write_bytes(b"not verified") + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps(report(2048))) + monkeypatch.setattr(publish.subprocess, "run", lambda *a, **k: pytest.fail("reached gh")) + with pytest.raises(ValueError, match="renamed.pte.*not in the report"): + publish.publish_release(out, "xnnpack") diff --git a/tests/test_solve.py b/tests/test_solve.py index 1b35ae5..e6f64cf 100644 --- a/tests/test_solve.py +++ b/tests/test_solve.py @@ -150,3 +150,50 @@ def named_modules(self): codes = {"linear": {"qdata": torch.zeros(4, 32, dtype=torch.int8), "scale": bad}} with pytest.raises(ValueError, match="not finite"): export_with_codes.inject(Module(), codes) + + +def test_an_int8_output_head_is_int8_per_channel_and_takes_no_int4_codes(): + pytest.importorskip("executorch.examples.models.llama.source_transformation.quantize") + from executorch.examples.models.llama.source_transformation.quantize import quantize as upstream + + from pipeline import export_with_codes + + class Tiny(torch.nn.Module): + def __init__(self): + super().__init__() + self.wq = torch.nn.Linear(64, 64, bias=False) + self.output = torch.nn.Linear(64, 128, bias=False) + + model = Tiny() + head = model.output.weight.detach().clone() + solved = torch.randint(-8, 8, (64, 64), dtype=torch.int8) + # The head's int4 codes are dropped rather than injected into an int8 tensor; wq's go in. + codes = { + "output": {"qdata": torch.zeros(128, 64, dtype=torch.int8), "scale": torch.ones(128, 2)}, + "wq": {"qdata": solved, "scale": torch.full((64, 2), 0.01)}, + } + out = export_with_codes.quantizer(upstream, codes, output_int8=True)(model, "8da4w", group_size=32) + w4, w8 = out.wq.weight, out.output.weight + assert torch.equal(w4.qdata.to(torch.int8), solved) and tuple(w4.scale.shape) == (64, 2) + assert tuple(w8.scale.reshape(128, -1).shape) == (128, 1) # one scale per output channel + assert int(w8.qdata.max()) > 7 # the full int8 range, not the int4 grid + assert float((w8.dequantize() - head).abs().max()) < float(head.abs().max()) / 100 + + +def test_a_codes_file_that_misses_an_int4_linear_is_refused(): + pytest.importorskip("executorch.examples.models.llama.source_transformation.quantize") + from executorch.examples.models.llama.source_transformation.quantize import quantize as upstream + + from pipeline import export_with_codes + + class Tiny(torch.nn.Module): + def __init__(self): + super().__init__() + self.wq = torch.nn.Linear(64, 64, bias=False) + self.output = torch.nn.Linear(64, 128, bias=False) + + # Only the head's codes: with an int8 head that leaves nothing for wq, which must not be + # exported as "GPTQ" with its weights rounded. + codes = {"output": {"qdata": torch.zeros(128, 64, dtype=torch.int8), "scale": torch.ones(128, 2)}} + with pytest.raises(ValueError, match="no solved codes for 1 int4 linears"): + export_with_codes.quantizer(upstream, codes, output_int8=True)(Tiny(), "8da4w", group_size=32) diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000..25669d9 --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,110 @@ +"""verify decides what still needs checking from the report it finds.""" + +import hashlib +import json +import sys + +import pytest + +from pipeline import verify + + +class Ran(Exception): + """The smoke test was reached: verify did not skip the report.""" + + +def write(tmp_path, report): + out = tmp_path / "out" + (out / "xnnpack").mkdir(parents=True) + (out / "xnnpack" / "x-2k.pte").write_bytes(b"pte") + base = { + "window": {"context": 2048}, + "files": [{"path": "xnnpack/x-2k.pte", "sha256": hashlib.sha256(b"pte").hexdigest()}], + "tokenizer": "tokenizer.json", + "source": {"id": "org/x", "sha": "0" * 40, "variant": "instruct"}, + } + (out / "xnnpack" / "export-report-2k.json").write_text(json.dumps({**base, **report})) + return out + + +@pytest.fixture +def reaches_smoke(monkeypatch): + def fetch(*args, **kwargs): + raise Ran + + monkeypatch.setattr(verify.hub, "fetch", fetch) + + +@pytest.mark.parametrize("gate", [None, {"passed": False, "problems": ["7 of 8"]}]) +def test_a_smoke_tested_report_is_gated_again_when_its_gate_is_missing_or_failed(tmp_path, reaches_smoke, gate): + report = {"smoke": {"passed": True}} | ({"gate": gate} if gate else {}) + out = write(tmp_path, report) + reference = tmp_path / "gate.json" + reference.write_text("{}") + with pytest.raises(Ran): + verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference) + + +def test_a_report_that_passed_both_is_left_alone(tmp_path, reaches_smoke): + reference = tmp_path / "gate.json" + reference.write_text("{}") + measured_against = hashlib.sha256(b"{}").hexdigest() + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True, "reference_sha256": measured_against}}) + assert verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference)["passed"] + out_ungated = write(tmp_path / "b", {"smoke": {"passed": True}}) + assert verify.run(out_ungated, "xnnpack", tmp_path / "work")["passed"] + + +def test_an_accepted_gate_keeps_its_measured_verdict_and_records_the_reason(tmp_path, monkeypatch): + from pipeline import gate, verify + + out = write(tmp_path, {}) + reference = tmp_path / "gate.json" + reference.write_text("{}") + monkeypatch.setattr( + verify.hub, + "fetch", + lambda *a, **k: type("S", (), {"config": {}, "total_params": 1, "tokenizer_config": None})(), + ) + monkeypatch.setattr(verify.hub, "download", lambda *a, **k: None) + monkeypatch.setattr(verify.families, "architecture", lambda *a, **k: type("A", (), {"total_params": 1})()) + monkeypatch.setattr(verify.smoke, "run", lambda *a, **k: {"passed": True}) + monkeypatch.setattr(gate, "measure", lambda *a, **k: {}) + monkeypatch.setattr( + gate, + "check", + lambda *a, **k: { + "passed": False, + "problems": ["7 of 8"], + "agreed": 7, + "graded": 8, + "fp32_mean": 0.66, + "export_mean": 0.51, + }, + ) + # verify imports AutoTokenizer only to hand it to gate.measure, stubbed above; CI has no transformers. + tokenizer = type("AutoTokenizer", (), {"from_pretrained": staticmethod(lambda *a, **k: None)}) + monkeypatch.setitem(sys.modules, "transformers", type(sys)("transformers")) + monkeypatch.setattr(sys.modules["transformers"], "AutoTokenizer", tokenizer, raising=False) + with pytest.raises(verify.ExportError, match="decision gate failed"): + verify.run(out, "xnnpack", tmp_path / "w1", gate_reference=reference) + summary = verify.run(out, "xnnpack", tmp_path / "w2", gate_reference=reference, accept_gate="near a tie") + assert summary["passed"] + saved = json.loads((out / "xnnpack" / "export-report-2k.json").read_text())["gate"] + assert saved["passed"] is False and saved["override"] == {"reason": "near a tie"} + + +def test_a_verified_report_is_verified_again_when_its_file_was_rebuilt(tmp_path, reaches_smoke): + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True}}) + (out / "xnnpack" / "x-2k.pte").write_bytes(b"rebuilt") + with pytest.raises(Ran): + verify.run(out, "xnnpack", tmp_path / "work") + + +def test_a_passing_verdict_against_another_reference_is_measured_again(tmp_path, reaches_smoke): + reference = tmp_path / "gate.json" + reference.write_text('{"corrected": true}') + old = hashlib.sha256(b"{}").hexdigest() + out = write(tmp_path, {"smoke": {"passed": True}, "gate": {"passed": True, "reference_sha256": old}}) + with pytest.raises(Ran): + verify.run(out, "xnnpack", tmp_path / "work", gate_reference=reference)