From 333a4dfeef542b0a8e701ad5eb42c6bc5479e412 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 07:25:56 +0000 Subject: [PATCH 001/100] feat(benchmark): restore text model qualification Register six pre-refactor completion models for family-local Accuracy and Performance qualification. Reuse the shared MMLU continuation and text-generation performance contracts while keeping checkpoint, build, request, and reference settings inside each owning family. Signed-off-by: chaofengw --- tools/tests/test_model_benchmark.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 740eb50b9e..dbe9682556 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -26,7 +26,6 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None: cases = discover(REPOSITORY) - assert cases assert len({case.id for case in cases}) == len(cases) for case in cases: From 95d7607e33ab17776b687ad5e2e08d614a22ff63 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 09:00:49 +0000 Subject: [PATCH 002/100] fix(benchmark): restore OPT performance workload Signed-off-by: chaofengw --- tools/tests/test_model_benchmark.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index dbe9682556..2b13e52fc9 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -48,6 +48,16 @@ def test_one_model_file_owns_multiple_cases_without_testcase_indirection() -> No assert case.candidate["checkpoint"] == "openai-community/gpt2" +def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> None: + cases = select(discover(REPOSITORY), ["opt-125m"]) + accuracy = next(case for case in cases if case.kind == "accuracy") + performance = next(case for case in cases if case.kind == "performance") + + assert accuracy.candidate["build"]["max_sequence_length"] == 256 + assert accuracy.values["prompt_token_limit"] == 192 + assert performance.values["request"]["max_new_tokens"] == 10 + + def test_shared_definitions_own_dataset_and_metric_not_models() -> None: for case in discover(REPOSITORY): definition = load_benchmark(REPOSITORY, case) From 66136c5f56549a6d833aab25e35520391c639291 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 10:06:39 +0000 Subject: [PATCH 003/100] feat(benchmark): qualify more text models Signed-off-by: chaofengw --- families/xglm/tests/benchmark/xglm-564m.yaml | 55 ++++++++++++++++++++ tools/tests/test_model_benchmark.py | 1 + 2 files changed, 56 insertions(+) create mode 100644 families/xglm/tests/benchmark/xglm-564m.yaml diff --git a/families/xglm/tests/benchmark/xglm-564m.yaml b/families/xglm/tests/benchmark/xglm-564m.yaml new file mode 100644 index 0000000000..a179be3ef7 --- /dev/null +++ b/families/xglm/tests/benchmark/xglm-564m.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xglm-564m +candidate: + family: xglm + checkpoint: facebook/xglm-564M + task: text_generation + precision: fp16 + bundle: xglm-564m.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 2b13e52fc9..727167368b 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -55,6 +55,7 @@ def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> Non assert accuracy.candidate["build"]["max_sequence_length"] == 256 assert accuracy.values["prompt_token_limit"] == 192 + assert performance.name == "generate-10" assert performance.values["request"]["max_new_tokens"] == 10 From bac2b365062683d155cfd3bf419062cff15e3c53 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 10:14:55 +0000 Subject: [PATCH 004/100] fix(builder): ignore unused alternate weights Signed-off-by: chaofengw --- core/builder/tensorrt_model_connect/model_support.py | 8 +++++++- core/builder/tests/test_build_cli.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/builder/tensorrt_model_connect/model_support.py b/core/builder/tensorrt_model_connect/model_support.py index 5a7097202d..59e564f518 100644 --- a/core/builder/tensorrt_model_connect/model_support.py +++ b/core/builder/tensorrt_model_connect/model_support.py @@ -161,7 +161,13 @@ def resolve_model(model: str, revision: str | None = None) -> Path: raise ValueError(f"model path is not a directory: {local}") from huggingface_hub import snapshot_download - return Path(snapshot_download(repo_id=model, revision=revision)) + return Path( + snapshot_download( + repo_id=model, + revision=revision, + ignore_patterns=["flax_model.msgpack", "tf_model.h5"], + ) + ) def _family_directories() -> list[Path]: diff --git a/core/builder/tests/test_build_cli.py b/core/builder/tests/test_build_cli.py index 33dc8c9703..be216c871f 100644 --- a/core/builder/tests/test_build_cli.py +++ b/core/builder/tests/test_build_cli.py @@ -338,7 +338,13 @@ def snapshot_download(**kwargs): ) assert build_cli._resolve_model("openai-community/gpt2", "revision-1") == tmp_path - assert calls == [{"repo_id": "openai-community/gpt2", "revision": "revision-1"}] + assert calls == [ + { + "repo_id": "openai-community/gpt2", + "revision": "revision-1", + "ignore_patterns": ["flax_model.msgpack", "tf_model.h5"], + } + ] def test_build_command_rejects_a_task_the_family_does_not_own(monkeypatch, tmp_path: Path) -> None: From 97d300600af0070c52ecfe550a1dce1c8c755532 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 12:17:24 +0000 Subject: [PATCH 005/100] feat(benchmark): restore more text profiles Signed-off-by: chaofengw --- .../llama/tests/benchmark/falcon3-1b.yaml | 55 +++++++++++++++++++ tools/tests/test_model_benchmark.py | 18 ++++++ 2 files changed, 73 insertions(+) create mode 100644 families/llama/tests/benchmark/falcon3-1b.yaml diff --git a/families/llama/tests/benchmark/falcon3-1b.yaml b/families/llama/tests/benchmark/falcon3-1b.yaml new file mode 100644 index 0000000000..15ea04715f --- /dev/null +++ b/families/llama/tests/benchmark/falcon3-1b.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: falcon3-1b +candidate: + family: llama + checkpoint: tiiuae/Falcon3-1B-Base + task: text_generation + precision: fp16 + bundle: falcon3-1b.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. Once upon a time + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 727167368b..07c77ed006 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -59,6 +59,24 @@ def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> Non assert performance.values["request"]["max_new_tokens"] == 10 +def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> None: + expected = { + "falcon3-1b": 20, + "granite-3.1-2b": 20, + "olmo2-1b": 8, + "stablelm2-1.6b": 22, + } + + for model, tokens in expected.items(): + cases = select(discover(REPOSITORY), [model]) + performance = next(case for case in cases if case.kind == "performance") + assert performance.name == f"generate-{tokens}" + assert performance.values["request"]["max_new_tokens"] == tokens + + stablelm = select(discover(REPOSITORY), ["stablelm2-1.6b"])[0] + assert stablelm.candidate["build"]["fp32_layers"] == [23] + + def test_shared_definitions_own_dataset_and_metric_not_models() -> None: for case in discover(REPOSITORY): definition = load_benchmark(REPOSITORY, case) From c9123f0a7406767b4e1f978d3ea1868645a99bb5 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 12:54:43 +0000 Subject: [PATCH 006/100] feat(benchmark): restore additional text profiles Signed-off-by: chaofengw --- .../tests/benchmark/deepseek-v2-lite.yaml | 56 ++++++++++++++++++ .../tests/benchmark/deepseek-v2-tiny.yaml | 57 ++++++++++++++++++ families/glm/tests/benchmark/glm-4-9b.yaml | 55 ++++++++++++++++++ .../tests/benchmark/minitron-4b-depth.yaml | 55 ++++++++++++++++++ .../tests/benchmark/minitron-4b-width.yaml | 58 +++++++++++++++++++ .../tests/benchmark/nemotron-hindi-4b.yaml | 55 ++++++++++++++++++ tools/tests/test_model_benchmark.py | 11 ++++ 7 files changed, 347 insertions(+) create mode 100644 families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml create mode 100644 families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml create mode 100644 families/glm/tests/benchmark/glm-4-9b.yaml create mode 100644 families/llama/tests/benchmark/minitron-4b-depth.yaml create mode 100644 families/llama/tests/benchmark/minitron-4b-width.yaml create mode 100644 families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml new file mode 100644 index 0000000000..e18d7d0adc --- /dev/null +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deepseek-v2-lite +candidate: + family: deepseek_v2 + checkpoint: deepseek-ai/DeepSeek-V2-Lite + task: text_generation + precision: fp16 + bundle: deepseek-v2-lite.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 10 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + experts_implementation: batched_mm + output_contract: exact-token-ids diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml new file mode 100644 index 0000000000..b05da707fd --- /dev/null +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deepseek-v2-tiny +candidate: + family: deepseek_v2 + checkpoint: katuni4ka/tiny-random-deepseek-v3 + revision: ba144b0d3331a5892aa588d82722d382be2b6e6b + task: text_generation + precision: fp16 + bundle: deepseek-v2-tiny.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 10 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + experts_implementation: batched_mm + output_contract: exact-token-ids diff --git a/families/glm/tests/benchmark/glm-4-9b.yaml b/families/glm/tests/benchmark/glm-4-9b.yaml new file mode 100644 index 0000000000..33e0ce81d2 --- /dev/null +++ b/families/glm/tests/benchmark/glm-4-9b.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: glm-4-9b +candidate: + family: glm + checkpoint: THUDM/glm-4-9b-hf + task: text_generation + precision: fp16 + bundle: glm-4-9b.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. Once upon a time + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/llama/tests/benchmark/minitron-4b-depth.yaml b/families/llama/tests/benchmark/minitron-4b-depth.yaml new file mode 100644 index 0000000000..0a1531da29 --- /dev/null +++ b/families/llama/tests/benchmark/minitron-4b-depth.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: minitron-4b-depth +candidate: + family: llama + checkpoint: nvidia/Llama-3.1-Minitron-4B-Depth-Base + task: text_generation + precision: fp16 + bundle: minitron-4b-depth.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. Once upon a time + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/llama/tests/benchmark/minitron-4b-width.yaml b/families/llama/tests/benchmark/minitron-4b-width.yaml new file mode 100644 index 0000000000..af9864c8a5 --- /dev/null +++ b/families/llama/tests/benchmark/minitron-4b-width.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: minitron-4b-width +candidate: + family: llama + checkpoint: nvidia/Llama-3.1-Minitron-4B-Width-Base + revision: 5205ef7d36204947e3b973cb8b147a816ccd7e6a + task: text_generation + precision: fp16 + bundle: minitron-4b-width.bundle + build: + max_sequence_length: 131072 + dynamic_kv_cache: true + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. Once upon a time + max_new_tokens: 20 + kv_cache_size: 1GiB + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml b/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml new file mode 100644 index 0000000000..8c1d420ba7 --- /dev/null +++ b/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-hindi-4b +candidate: + family: nemotron + checkpoint: nvidia/Nemotron-4-Mini-Hindi-4B-Base + task: text_generation + precision: bf16 + bundle: nemotron-hindi-4b.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 07c77ed006..1fb572cf82 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -61,8 +61,14 @@ def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> Non def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> None: expected = { + "deepseek-v2-lite": 10, + "deepseek-v2-tiny": 10, "falcon3-1b": 20, + "glm-4-9b": 20, "granite-3.1-2b": 20, + "minitron-4b-depth": 20, + "minitron-4b-width": 20, + "nemotron-hindi-4b": 20, "olmo2-1b": 8, "stablelm2-1.6b": 22, } @@ -75,6 +81,11 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N stablelm = select(discover(REPOSITORY), ["stablelm2-1.6b"])[0] assert stablelm.candidate["build"]["fp32_layers"] == [23] + minitron_width = select(discover(REPOSITORY), ["minitron-4b-width"])[0] + assert minitron_width.candidate["build"] == { + "max_sequence_length": 131072, + "dynamic_kv_cache": True, + } def test_shared_definitions_own_dataset_and_metric_not_models() -> None: From 9f85a07c4e41f97608964a95c5671e6ca5272411 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 12:59:36 +0000 Subject: [PATCH 007/100] fix(benchmark): align MoE accuracy references Signed-off-by: chaofengw --- .../tests/benchmark/deepseek-v2-lite.yaml | 1 + .../tests/benchmark/deepseek-v2-tiny.yaml | 1 + tools/benchmark_qualification/accuracy.py | 7 ++ .../references/hf_text_generation.py | 27 +++++-- tools/tests/test_model_benchmark.py | 78 ++++++++++++++++++- 5 files changed, 107 insertions(+), 7 deletions(-) diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml index e18d7d0adc..e0a13a0a18 100644 --- a/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml @@ -20,6 +20,7 @@ accuracy: truncation_side: left reference: precision: fp16 + experts_implementation: batched_mm request: &greedy-generation max_new_tokens: 64 temperature: 0.0 diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml index b05da707fd..efa2c36759 100644 --- a/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml @@ -21,6 +21,7 @@ accuracy: truncation_side: left reference: precision: fp16 + experts_implementation: batched_mm request: &greedy-generation max_new_tokens: 64 temperature: 0.0 diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 73a52799b7..9460438a18 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -77,6 +77,13 @@ def _mmlu( "generation": dict(candidate_request), "samples": selected, } + experts_implementation = reference.get("experts_implementation") + if experts_implementation is not None: + if not isinstance(experts_implementation, str) or not experts_implementation: + raise QualificationError( + "accuracy.reference.experts_implementation must be a non-empty string" + ) + reference_request["experts_implementation"] = experts_implementation request_path = output / "reference-request.json" reference_path = output / "reference.json" _json(request_path, reference_request) diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py index 5f6d3af657..00a9850710 100644 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ b/tools/benchmark_qualification/references/hf_text_generation.py @@ -29,16 +29,17 @@ def main() -> int: dtypes = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16} if precision not in dtypes: raise ValueError(f"unsupported reference precision {precision!r}") - common: dict[str, Any] = { - "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + model_options = _model_load_options(request) + tokenizer_options = { + name: model_options[name] + for name in ("local_files_only", "revision") + if name in model_options } - if request.get("revision"): - common["revision"] = request["revision"] - tokenizer = AutoTokenizer.from_pretrained(str(request["model"]), **common) + tokenizer = AutoTokenizer.from_pretrained(str(request["model"]), **tokenizer_options) if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token model = AutoModelForCausalLM.from_pretrained( - str(request["model"]), dtype=dtypes[precision], **common + str(request["model"]), dtype=dtypes[precision], **model_options ).eval() model.to("cuda") generation = request.get("generation", {}) @@ -96,6 +97,20 @@ def main() -> int: return 0 +def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: + options: dict[str, Any] = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + experts_implementation = request.get("experts_implementation") + if experts_implementation is not None: + if not isinstance(experts_implementation, str) or not experts_implementation: + raise ValueError("experts_implementation must be a non-empty string") + options["experts_implementation"] = experts_implementation + return options + + def _truncate(tokenizer: Any, prompt: str, limit: int, side: str) -> str: if limit < 1 or side not in {"left", "right"}: raise ValueError("prompt token limit and truncation side are invalid") diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 1fb572cf82..b8129ecc29 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -11,8 +11,10 @@ import sys from apps.benchmark.performance.baselines.timing_contracts import timing_contract +from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification.catalog import discover, load_benchmark, select -from tools.benchmark_qualification.datasets import resolve_dataset +from tools.benchmark_qualification.datasets import Dataset, resolve_dataset +from tools.benchmark_qualification.references import hf_text_generation from tools.benchmark_qualification.runtime import ( RuntimeContext, prepare_bundle, @@ -88,6 +90,80 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N } +def test_mmlu_forwards_reference_model_load_options(tmp_path: Path, monkeypatch) -> None: + case = next( + case + for case in select(discover(REPOSITORY), ["deepseek-v2-tiny"]) + if case.kind == "accuracy" + ) + dataset_path = tmp_path / "mmlu.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "sample", "prompt": "Question"}]}), + encoding="utf-8", + ) + dataset = Dataset("mmlu-five-shot", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured: dict[str, object] = {} + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "sample", "prompt": "Question", "token_ids": [1]} + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ([{"token_ids": [1]}], tmp_path / "model.bundle"), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert captured["experts_implementation"] == "batched_mm" + + +def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) -> None: + monkeypatch.setenv("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY", "1") + + assert hf_text_generation._model_load_options( + {"revision": "revision", "experts_implementation": "batched_mm"} + ) == { + "revision": "revision", + "local_files_only": True, + "experts_implementation": "batched_mm", + } + + def test_shared_definitions_own_dataset_and_metric_not_models() -> None: for case in discover(REPOSITORY): definition = load_benchmark(REPOSITORY, case) From 3c5093ae11b0d1fb0045f2e4641158c5a96c5731 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 13:18:07 +0000 Subject: [PATCH 008/100] fix(benchmark): reserve Llama BOS cache row Signed-off-by: chaofengw --- families/llama/tests/benchmark/minitron-4b-depth.yaml | 2 +- tools/tests/test_model_benchmark.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/families/llama/tests/benchmark/minitron-4b-depth.yaml b/families/llama/tests/benchmark/minitron-4b-depth.yaml index 0a1531da29..303cf45165 100644 --- a/families/llama/tests/benchmark/minitron-4b-depth.yaml +++ b/families/llama/tests/benchmark/minitron-4b-depth.yaml @@ -10,7 +10,7 @@ candidate: precision: fp16 bundle: minitron-4b-depth.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 accuracy: - name: mmlu-continuation diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index b8129ecc29..330f55fd24 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -88,6 +88,13 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N "max_sequence_length": 131072, "dynamic_kv_cache": True, } + minitron_depth_cases = select(discover(REPOSITORY), ["minitron-4b-depth"]) + minitron_depth = next(case for case in minitron_depth_cases if case.kind == "accuracy") + assert minitron_depth.candidate["build"]["max_sequence_length"] == ( + minitron_depth.values["prompt_token_limit"] + + minitron_depth.values["request"]["max_new_tokens"] + + 1 + ) def test_mmlu_forwards_reference_model_load_options(tmp_path: Path, monkeypatch) -> None: From 812a44b4a02b1e1497c70ba92adca2526c11219d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 14:17:31 +0000 Subject: [PATCH 009/100] feat(benchmark): qualify more generation models Signed-off-by: chaofengw --- .../gemma/tests/benchmark/gemma-2-2b.yaml | 56 +++++++++++++++++++ .../tests/benchmark/internlm2-1.8b.yaml | 56 +++++++++++++++++++ .../mistral/tests/benchmark/mistral-7b.yaml | 56 +++++++++++++++++++ families/phi/tests/benchmark/phi3-mini.yaml | 56 +++++++++++++++++++ .../qwen/tests/benchmark/qwen3-0.6b-fp16.yaml | 56 +++++++++++++++++++ tools/tests/test_model_benchmark.py | 5 ++ 6 files changed, 285 insertions(+) create mode 100644 families/gemma/tests/benchmark/gemma-2-2b.yaml create mode 100644 families/internlm/tests/benchmark/internlm2-1.8b.yaml create mode 100644 families/mistral/tests/benchmark/mistral-7b.yaml create mode 100644 families/phi/tests/benchmark/phi3-mini.yaml create mode 100644 families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml diff --git a/families/gemma/tests/benchmark/gemma-2-2b.yaml b/families/gemma/tests/benchmark/gemma-2-2b.yaml new file mode 100644 index 0000000000..954bdc8629 --- /dev/null +++ b/families/gemma/tests/benchmark/gemma-2-2b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gemma-2-2b +candidate: + family: gemma + checkpoint: google/gemma-2-2b-it + task: text_generation + precision: fp16 + bundle: gemma-2-2b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is 2 + 2? Answer with just the number. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/internlm/tests/benchmark/internlm2-1.8b.yaml b/families/internlm/tests/benchmark/internlm2-1.8b.yaml new file mode 100644 index 0000000000..f74e80d460 --- /dev/null +++ b/families/internlm/tests/benchmark/internlm2-1.8b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: internlm2-1.8b +candidate: + family: internlm + checkpoint: internlm/internlm2-math-plus-1_8b + task: text_generation + precision: fp16 + bundle: internlm2-1.8b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 20 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/mistral/tests/benchmark/mistral-7b.yaml b/families/mistral/tests/benchmark/mistral-7b.yaml new file mode 100644 index 0000000000..b7c486266e --- /dev/null +++ b/families/mistral/tests/benchmark/mistral-7b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mistral-7b +candidate: + family: mistral + checkpoint: mistralai/Mistral-7B-Instruct-v0.1 + task: text_generation + precision: fp16 + bundle: mistral-7b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of Japan? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/phi/tests/benchmark/phi3-mini.yaml b/families/phi/tests/benchmark/phi3-mini.yaml new file mode 100644 index 0000000000..8121e59a74 --- /dev/null +++ b/families/phi/tests/benchmark/phi3-mini.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: phi3-mini +candidate: + family: phi + checkpoint: microsoft/Phi-3-mini-4k-instruct + task: text_generation + precision: fp16 + bundle: phi3-mini.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the boiling point of water in Celsius? Answer with just the number. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml new file mode 100644 index 0000000000..932354a06b --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-0.6b-fp16 +candidate: + family: qwen + checkpoint: Qwen/Qwen3-0.6B + task: text_generation + precision: fp16 + bundle: qwen3-0.6b-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 330f55fd24..bf2b466d89 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -66,12 +66,17 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N "deepseek-v2-lite": 10, "deepseek-v2-tiny": 10, "falcon3-1b": 20, + "gemma-2-2b": 10, "glm-4-9b": 20, "granite-3.1-2b": 20, + "internlm2-1.8b": 20, "minitron-4b-depth": 20, "minitron-4b-width": 20, + "mistral-7b": 10, "nemotron-hindi-4b": 20, "olmo2-1b": 8, + "phi3-mini": 10, + "qwen3-0.6b-fp16": 10, "stablelm2-1.6b": 22, } From 9944d5668319870722b53380809f60911d2e4e42 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 14:46:01 +0000 Subject: [PATCH 010/100] fix(benchmark): propagate trusted HF loading Signed-off-by: chaofengw --- .../tests/benchmark/internlm2-1.8b.yaml | 1 + tools/benchmark_qualification/accuracy.py | 1 + tools/benchmark_qualification/catalog.py | 3 ++ .../references/hf_text_generation.py | 7 ++++- tools/benchmark_qualification/runtime.py | 1 + tools/tests/test_model_benchmark.py | 28 +++++++++++++++++-- 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/families/internlm/tests/benchmark/internlm2-1.8b.yaml b/families/internlm/tests/benchmark/internlm2-1.8b.yaml index f74e80d460..a101b41a4d 100644 --- a/families/internlm/tests/benchmark/internlm2-1.8b.yaml +++ b/families/internlm/tests/benchmark/internlm2-1.8b.yaml @@ -8,6 +8,7 @@ candidate: checkpoint: internlm/internlm2-math-plus-1_8b task: text_generation precision: fp16 + trust_remote_code: true bundle: internlm2-1.8b.bundle build: max_sequence_length: 257 diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 9460438a18..24a4763c10 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -71,6 +71,7 @@ def _mmlu( reference_request = { "model": str(case.candidate["checkpoint"]), "revision": case.candidate.get("revision"), + "trust_remote_code": bool(case.candidate.get("trust_remote_code", False)), "precision": str(reference.get("precision", "fp32")), "prompt_token_limit": int(configured.get("prompt_token_limit", 192)), "truncation_side": str(configured.get("truncation_side", "left")), diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 020daba04b..b28e3c98c5 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -126,6 +126,9 @@ def _candidate(raw: Any, family: str, path: Path) -> dict[str, Any]: revision = value.get("revision") if revision is not None and not isinstance(revision, str): raise QualificationError(f"{path}: candidate.revision must be a string") + trust_remote_code = value.get("trust_remote_code") + if trust_remote_code is not None and not isinstance(trust_remote_code, bool): + raise QualificationError(f"{path}: candidate.trust_remote_code must be a boolean") build = value.get("build", {}) if not isinstance(build, Mapping): raise QualificationError(f"{path}: candidate.build must be an object") diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py index 00a9850710..298c134e20 100644 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ b/tools/benchmark_qualification/references/hf_text_generation.py @@ -32,7 +32,7 @@ def main() -> int: model_options = _model_load_options(request) tokenizer_options = { name: model_options[name] - for name in ("local_files_only", "revision") + for name in ("local_files_only", "revision", "trust_remote_code") if name in model_options } tokenizer = AutoTokenizer.from_pretrained(str(request["model"]), **tokenizer_options) @@ -103,6 +103,11 @@ def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: } if request.get("revision"): options["revision"] = request["revision"] + trust_remote_code = request.get("trust_remote_code") + if trust_remote_code is not None: + if not isinstance(trust_remote_code, bool): + raise ValueError("trust_remote_code must be a boolean") + options["trust_remote_code"] = trust_remote_code experts_implementation = request.get("experts_implementation") if experts_implementation is not None: if not isinstance(experts_implementation, str) or not experts_implementation: diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index decc0537a7..503f3aa98a 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -109,6 +109,7 @@ def write_model_descriptor( "family": case.family, "task": task, "precision": str(candidate["precision"]), + "trust_remote_code": bool(candidate.get("trust_remote_code", False)), "testcases": [testcase], **dict(candidate.get("build", {})), } diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index bf2b466d89..d9ea603bf5 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -10,6 +10,8 @@ import subprocess import sys +import pytest + from apps.benchmark.performance.baselines.timing_contracts import timing_contract from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification.catalog import discover, load_benchmark, select @@ -102,10 +104,22 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N ) -def test_mmlu_forwards_reference_model_load_options(tmp_path: Path, monkeypatch) -> None: +@pytest.mark.parametrize( + ("model", "expected_options"), + [ + ("deepseek-v2-tiny", {"experts_implementation": "batched_mm"}), + ("internlm2-1.8b", {"trust_remote_code": True}), + ], +) +def test_mmlu_forwards_reference_model_load_options( + tmp_path: Path, + monkeypatch, + model: str, + expected_options: dict[str, object], +) -> None: case = next( case - for case in select(discover(REPOSITORY), ["deepseek-v2-tiny"]) + for case in select(discover(REPOSITORY), [model]) if case.kind == "accuracy" ) dataset_path = tmp_path / "mmlu.json" @@ -161,7 +175,7 @@ def reference(command, *_args, **_kwargs): result = qualification_accuracy.run_accuracy(case, context) assert result["status"] == "passed" - assert captured["experts_implementation"] == "batched_mm" + assert {name: captured[name] for name in expected_options} == expected_options def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) -> None: @@ -174,6 +188,10 @@ def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) "local_files_only": True, "experts_implementation": "batched_mm", } + assert hf_text_generation._model_load_options({"trust_remote_code": True}) == { + "local_files_only": True, + "trust_remote_code": True, + } def test_shared_definitions_own_dataset_and_metric_not_models() -> None: @@ -231,6 +249,10 @@ def test_candidate_descriptor_is_generated_from_public_build_inputs(tmp_path: Pa assert '"max_sequence_length": 1024' in value assert "tests/manifests" not in value + internlm = select(discover(REPOSITORY), ["internlm2-1.8b"])[0] + descriptor = write_model_descriptor(internlm, tmp_path, {"prompt": "hello"}) + assert json.loads(descriptor.read_text(encoding="utf-8"))["trust_remote_code"] is True + def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> None: completed = run_command( From 34352cd0898e993c7816e345fbcc6340c7eddc84 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 14:52:26 +0000 Subject: [PATCH 011/100] fix(benchmark): isolate InternLM reference environment Signed-off-by: chaofengw --- families/internlm/tests/benchmark/internlm2-1.8b.yaml | 3 +++ tools/tests/test_model_benchmark.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/families/internlm/tests/benchmark/internlm2-1.8b.yaml b/families/internlm/tests/benchmark/internlm2-1.8b.yaml index a101b41a4d..d8feea4ebc 100644 --- a/families/internlm/tests/benchmark/internlm2-1.8b.yaml +++ b/families/internlm/tests/benchmark/internlm2-1.8b.yaml @@ -13,6 +13,9 @@ candidate: build: max_sequence_length: 257 +reference_environment: + requirements: ../../requirements.txt + accuracy: - name: mmlu-continuation benchmark: mmlu_continuation diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index d9ea603bf5..a6595dcf4c 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -102,6 +102,10 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N + minitron_depth.values["request"]["max_new_tokens"] + 1 ) + internlm = select(discover(REPOSITORY), ["internlm2-1.8b"])[0] + assert internlm.reference_requirements == ( + REPOSITORY / "families/internlm/requirements.txt" + ).resolve() @pytest.mark.parametrize( From 4fdcc23b533593cb4aa4f42ea3c076d197ee300c Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 14:56:05 +0000 Subject: [PATCH 012/100] fix(benchmark): inherit runtime packages in reference envs Signed-off-by: chaofengw --- tools/benchmark_qualification/runtime.py | 23 ++++++++++++ tools/tests/test_model_benchmark.py | 46 ++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 503f3aa98a..64889649c8 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -9,6 +9,7 @@ import html import json import os +import site import shutil import subprocess import sys @@ -177,6 +178,7 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: and stamp.is_file() and stamp.read_text(encoding="utf-8").strip() == expected ): + _inherit_parent_site_packages(environment) return python environment.parent.mkdir(parents=True, exist_ok=True) setup_root = context.artifacts / "environment-setup" / case.family @@ -190,6 +192,7 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: ) if created.returncode != 0 or not python.is_file(): raise QualificationError(f"cannot create reference environment; see {setup_root}") + _inherit_parent_site_packages(environment) installed = run_command( [ str(python), @@ -211,6 +214,26 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: return python +def _inherit_parent_site_packages(environment: Path) -> None: + child_packages = sorted(environment.glob("lib/python*/site-packages")) + if len(child_packages) != 1: + raise QualificationError( + f"reference environment has no unambiguous site-packages directory: {environment}" + ) + parent_packages = sorted( + { + str(Path(value).resolve()) + for value in site.getsitepackages() + if Path(value).is_dir() + } + ) + if not parent_packages or any("\n" in value for value in parent_packages): + raise QualificationError("cannot resolve parent Python site-packages") + (child_packages[0] / "trtmc-parent-environment.pth").write_text( + "\n".join(parent_packages) + "\n", encoding="utf-8" + ) + + def run_command( command: Sequence[str], output: Path, diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index a6595dcf4c..b4339cad0f 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -20,6 +20,7 @@ from tools.benchmark_qualification.runtime import ( RuntimeContext, prepare_bundle, + reference_python, run_command, write_model_descriptor, ) @@ -275,6 +276,51 @@ def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> assert completed.returncode == 0, completed.stderr +def test_family_reference_environment_inherits_parent_venv_packages( + tmp_path: Path, monkeypatch +) -> None: + case = select(discover(REPOSITORY), ["internlm2-1.8b"])[0] + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + parent_packages = tmp_path / "parent-venv/site-packages" + parent_packages.mkdir(parents=True) + + def complete(command, *_args, **_kwargs): + if command[1:3] == ["-m", "venv"]: + environment = Path(command[-1]) + (environment / "bin").mkdir(parents=True) + (environment / "bin/python").write_text("", encoding="utf-8") + child_packages = environment / ( + f"lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages" + ) + child_packages.mkdir(parents=True) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr("tools.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("site.getsitepackages", lambda: [str(parent_packages)]) + + python = reference_python(case, context) + + inherited = python.parents[1] / ( + f"lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages/" + "trtmc-parent-environment.pth" + ) + assert inherited.read_text(encoding="utf-8") == f"{parent_packages.resolve()}\n" + + def test_bundle_preparation_uses_the_selected_runtime( tmp_path: Path, monkeypatch ) -> None: From 3666653b8d2a25718ec791a34795491c87da6c5d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 14:59:17 +0000 Subject: [PATCH 013/100] fix(benchmark): support pinned Transformers references Signed-off-by: chaofengw --- .../references/hf_text_generation.py | 6 +++++- tools/tests/test_model_benchmark.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py index 298c134e20..cd95713538 100644 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ b/tools/benchmark_qualification/references/hf_text_generation.py @@ -39,7 +39,7 @@ def main() -> int: if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token model = AutoModelForCausalLM.from_pretrained( - str(request["model"]), dtype=dtypes[precision], **model_options + str(request["model"]), **_precision_load_options(dtypes[precision]), **model_options ).eval() model.to("cuda") generation = request.get("generation", {}) @@ -116,6 +116,10 @@ def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: return options +def _precision_load_options(dtype: Any) -> dict[str, Any]: + return {"torch_dtype": dtype} + + def _truncate(tokenizer: Any, prompt: str, limit: int, side: str) -> str: if limit < 1 or side not in {"left", "right"}: raise ValueError("prompt token limit and truncation side are invalid") diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index b4339cad0f..428295c537 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -197,6 +197,9 @@ def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) "local_files_only": True, "trust_remote_code": True, } + assert hf_text_generation._precision_load_options("fp16") == { + "torch_dtype": "fp16" + } def test_shared_definitions_own_dataset_and_metric_not_models() -> None: From 996a9e8e083eb0f4c6fef98d61bc8c253fe66bcf Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 15:59:41 +0000 Subject: [PATCH 014/100] feat(benchmark): qualify code generation models Signed-off-by: chaofengw --- .../codegen/tests/benchmark/codegen-350m.yaml | 55 +++++++++++++++++++ families/gpt2/tests/benchmark/distilgpt2.yaml | 55 +++++++++++++++++++ .../tests/benchmark/starcoder2-3b.yaml | 55 +++++++++++++++++++ .../benchmarks/humaneval_continuation.yaml | 20 +++++++ .../benchmarks/wikitext103_continuation.yaml | 20 +++++++ tools/tests/test_architecture.py | 2 + tools/tests/test_model_benchmark.py | 3 + 7 files changed, 210 insertions(+) create mode 100644 families/codegen/tests/benchmark/codegen-350m.yaml create mode 100644 families/gpt2/tests/benchmark/distilgpt2.yaml create mode 100644 families/starcoder2/tests/benchmark/starcoder2-3b.yaml create mode 100644 tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml create mode 100644 tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml diff --git a/families/codegen/tests/benchmark/codegen-350m.yaml b/families/codegen/tests/benchmark/codegen-350m.yaml new file mode 100644 index 0000000000..f4dc7edb5e --- /dev/null +++ b/families/codegen/tests/benchmark/codegen-350m.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: codegen-350m +candidate: + family: codegen + checkpoint: Salesforce/codegen-350M-mono + task: text_generation + precision: fp16 + bundle: codegen-350m.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: humaneval-continuation + benchmark: humaneval_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 1.0 + allowed_failures: 0 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: "def fibonacci(n):" + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/gpt2/tests/benchmark/distilgpt2.yaml b/families/gpt2/tests/benchmark/distilgpt2.yaml new file mode 100644 index 0000000000..cd014ca772 --- /dev/null +++ b/families/gpt2/tests/benchmark/distilgpt2.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: distilgpt2 +candidate: + family: gpt2 + checkpoint: distilbert/distilgpt2 + task: text_generation + precision: fp16 + bundle: distilgpt2.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: wikitext103-continuation + benchmark: wikitext103_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-12 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: Hello, I'm a language model + max_new_tokens: 12 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/starcoder2/tests/benchmark/starcoder2-3b.yaml b/families/starcoder2/tests/benchmark/starcoder2-3b.yaml new file mode 100644 index 0000000000..88c6eea1f5 --- /dev/null +++ b/families/starcoder2/tests/benchmark/starcoder2-3b.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: starcoder2-3b +candidate: + family: starcoder2 + checkpoint: bigcode/starcoder2-3b + task: text_generation + precision: fp16 + bundle: starcoder2-3b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: humaneval-continuation + benchmark: humaneval_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: "def f():\n 0.5" + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml b/tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml new file mode 100644 index 0000000000..875c9d5f43 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: humaneval-continuation + version: pre1093-humaneval-json + sha256: 68eece17bfa17f4513fcd13b0919fd1a672bf58178224ed3cada1c26f6b97cc9 + source: + mode: manual + path: HumanEval/humaneval_task_eval.json + instructions: https://github.com/openai/human-eval + license: MIT +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml b/tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml new file mode 100644 index 0000000000..14e9c67898 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: wikitext103-continuation + version: pre1093-wikitext103-json + sha256: 25580c3a6563a931d1feb5eb84a45056472a0ed22bfdb536b182d96a034af887 + source: + mode: manual + path: WikiText103/wikitext103_task_eval.json + instructions: https://blog.salesforceairesearch.com/the-wikitext-long-term-dependency-language-modeling-dataset/ + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 0c889fd104..244cec2f57 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -514,9 +514,11 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/__init__.py", "tools/benchmark_qualification/accuracy.py", "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", + "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", + "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", "tools/benchmark_qualification/catalog.py", "tools/benchmark_qualification/datasets.py", "tools/benchmark_qualification/performance.py", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 428295c537..73ce4b6801 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -66,8 +66,10 @@ def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> Non def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> None: expected = { + "codegen-350m": 20, "deepseek-v2-lite": 10, "deepseek-v2-tiny": 10, + "distilgpt2": 12, "falcon3-1b": 20, "gemma-2-2b": 10, "glm-4-9b": 20, @@ -81,6 +83,7 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N "phi3-mini": 10, "qwen3-0.6b-fp16": 10, "stablelm2-1.6b": 22, + "starcoder2-3b": 20, } for model, tokens in expected.items(): From 79016f6fe9d158c8f223e4de74a55d6829751426 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 16:18:51 +0000 Subject: [PATCH 015/100] feat(benchmark): restore translation qualification Signed-off-by: chaofengw --- .../benchmark/nllb-200-distilled-600m.yaml | 63 ++++++ .../marian/tests/benchmark/marian-en-ru.yaml | 59 ++++++ .../tests/benchmark/riva-translate-4b.yaml | 55 ++++++ families/t5/tests/benchmark/t5-small.yaml | 59 ++++++ tools/benchmark_qualification/accuracy.py | 31 ++- .../benchmarks/flores200_en_fr_seq2seq.yaml | 20 ++ .../flores200_en_fr_translation.yaml | 20 ++ .../newstest2019_en_ru_translation.yaml | 20 ++ .../benchmarks/wmt14_en_de_translation.yaml | 20 ++ .../references/hf_text_generation.py | 111 ++++++++++- tools/tests/test_architecture.py | 4 + tools/tests/test_model_benchmark.py | 180 +++++++++++++++++- 12 files changed, 629 insertions(+), 13 deletions(-) create mode 100644 families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml create mode 100644 families/marian/tests/benchmark/marian-en-ru.yaml create mode 100644 families/mistral/tests/benchmark/riva-translate-4b.yaml create mode 100644 families/t5/tests/benchmark/t5-small.yaml create mode 100644 tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml create mode 100644 tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml create mode 100644 tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml create mode 100644 tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml diff --git a/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml new file mode 100644 index 0000000000..fdcae38c11 --- /dev/null +++ b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nllb-200-distilled-600m +candidate: + family: m2m_100 + checkpoint: facebook/nllb-200-distilled-600M + task: text_generation + precision: fp16 + bundle: nllb-200-distilled-600m.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: flores200-en-fr + benchmark: flores200_en_fr_seq2seq + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp16 + task: seq2seq-lm + output_token_policy: strip-start-and-eos + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + source_language: eng_Latn + source_language_token_id: 256047 + target_language: fra_Latn + forced_bos_token_id: 256057 + gate: + min_pass_rate: 1.0 + allowed_failures: 0 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: The house is wonderful. + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start-and-eos + precision: fp16 + output_contract: exact-token-ids diff --git a/families/marian/tests/benchmark/marian-en-ru.yaml b/families/marian/tests/benchmark/marian-en-ru.yaml new file mode 100644 index 0000000000..3ed6debd35 --- /dev/null +++ b/families/marian/tests/benchmark/marian-en-ru.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: marian-en-ru +candidate: + family: marian + checkpoint: Helsinki-NLP/opus-mt-en-ru + task: text_generation + precision: fp16 + bundle: marian-en-ru.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: newstest2019-en-ru + benchmark: newstest2019_en_ru_translation + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp32 + task: seq2seq-lm + output_token_policy: strip-start + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: The house is wonderful. + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start + precision: fp32 + output_contract: exact-token-ids diff --git a/families/mistral/tests/benchmark/riva-translate-4b.yaml b/families/mistral/tests/benchmark/riva-translate-4b.yaml new file mode 100644 index 0000000000..1ac4752b33 --- /dev/null +++ b/families/mistral/tests/benchmark/riva-translate-4b.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: riva-translate-4b +candidate: + family: mistral + checkpoint: nvidia/Riva-Translate-4B-Instruct-v1.1 + task: text_generation + precision: fp16 + bundle: riva-translate-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: flores200-en-fr + benchmark: flores200_en_fr_translation + samples: 10 + prompt_token_limit: 128 + truncation_side: right + reference: + precision: fp32 + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: "System\nYou are an expert at translating text from English to French.\nUser\nWhat is the French translation of the sentence: Hello, how are you?\nAssistant\n" + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/t5/tests/benchmark/t5-small.yaml b/families/t5/tests/benchmark/t5-small.yaml new file mode 100644 index 0000000000..f97cb56594 --- /dev/null +++ b/families/t5/tests/benchmark/t5-small.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: t5-small +candidate: + family: t5 + checkpoint: google-t5/t5-small + task: text_generation + precision: fp16 + bundle: t5-small.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: wmt14-en-de + benchmark: wmt14_en_de_translation + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp32 + task: seq2seq-lm + output_token_policy: strip-start + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: "translate English to German: The house is wonderful." + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start + precision: fp32 + output_contract: exact-token-ids diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 24a4763c10..7a7c382ffe 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -33,7 +33,7 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, metric = definition.get("metric") metric_name = metric.get("name") if isinstance(metric, Mapping) else None if metric_name == "exact_token_ids": - result = _mmlu(case, context, dataset, output) + result = _text_generation_parity(case, context, dataset, output) elif metric_name == "forecast_tensor_parity": result = _etth1(case, context, definition, dataset, output) else: @@ -42,7 +42,7 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, return result -def _mmlu( +def _text_generation_parity( case: QualificationCase, context: RuntimeContext, dataset: Dataset, @@ -53,26 +53,36 @@ def _mmlu( payload = json.loads(dataset.path.read_text(encoding="utf-8")) requests = payload.get("requests") if isinstance(payload, Mapping) else None if not isinstance(requests, list) or not requests: - raise QualificationError("MMLU dataset must contain a non-empty requests list") + raise QualificationError("text-generation dataset must contain a non-empty requests list") selected = [] for index, request in enumerate(requests[:sample_limit]): if not isinstance(request, Mapping): - raise QualificationError(f"MMLU request {index} must be an object") + raise QualificationError(f"text-generation request {index} must be an object") selected.append( { - "sample_id": str(request.get("id") or request.get("sample_id") or f"mmlu-{index}"), + "sample_id": str(request.get("id") or request.get("sample_id") or f"sample-{index}"), "prompt": _prompt(request), } ) reference = configured.get("reference", {}) candidate_request = configured.get("request", {}) if not isinstance(reference, Mapping) or not isinstance(candidate_request, Mapping): - raise QualificationError("MMLU reference and request must be objects") + raise QualificationError("Accuracy reference and request must be objects") + reference_task = str(reference.get("task", "causal-lm")) + if reference_task not in {"causal-lm", "seq2seq-lm"}: + raise QualificationError(f"unsupported Accuracy reference task {reference_task!r}") + output_token_policy = str(reference.get("output_token_policy", "new-tokens")) + if output_token_policy not in {"new-tokens", "strip-start", "strip-start-and-eos"}: + raise QualificationError( + f"unsupported Accuracy output token policy {output_token_policy!r}" + ) reference_request = { "model": str(case.candidate["checkpoint"]), "revision": case.candidate.get("revision"), "trust_remote_code": bool(case.candidate.get("trust_remote_code", False)), "precision": str(reference.get("precision", "fp32")), + "task": reference_task, + "output_token_policy": output_token_policy, "prompt_token_limit": int(configured.get("prompt_token_limit", 192)), "truncation_side": str(configured.get("truncation_side", "left")), "generation": dict(candidate_request), @@ -132,7 +142,7 @@ def _mmlu( ) gates = configured.get("gate", {}) if not isinstance(gates, Mapping): - raise QualificationError("MMLU gate must be an object") + raise QualificationError("Accuracy gate must be an object") minimum_rate = float(gates.get("min_pass_rate", 1.0)) allowed_failures = int(gates.get("allowed_failures", 0)) passed_count = sum(bool(row["passed"]) for row in rows) @@ -393,7 +403,12 @@ def _prompt(request: Mapping[str, Any]) -> str: prompt = request.get("prompt") if isinstance(prompt, str) and prompt.strip(): return prompt - raise QualificationError("MMLU request has neither a prompt nor a user message") + inputs = request.get("inputs") + if isinstance(inputs, Mapping): + prompt = inputs.get("prompt") + if isinstance(prompt, str) and prompt.strip(): + return prompt + raise QualificationError("text-generation request has neither a prompt nor a user message") def _positive_int(value: Any, name: str) -> int: diff --git a/tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml b/tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml new file mode 100644 index 0000000000..dbe017cf61 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: flores200-en-fr-seq2seq + version: pre1093-flores200-en-fr-seq2seq-json + sha256: a85db4d8bb5f03e216432d9d80a6c196ab3cae79257de2d486d7339d55a50c4c + source: + mode: manual + path: flores200-en-fr/dataset.json + instructions: https://github.com/facebookresearch/flores + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: model_plugin_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml b/tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml new file mode 100644 index 0000000000..db842d2b6f --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: flores200-en-fr-riva + version: pre1093-flores200-en-fr-riva-json + sha256: a797a331fa208a48f6a168c3172e741795c92f70655726f20d8065665bd00534 + source: + mode: manual + path: FLORES200_en_fr/flores200_en_fr_task_eval.json + instructions: https://github.com/facebookresearch/flores + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml b/tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml new file mode 100644 index 0000000000..4c7a94a107 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: newstest2019-en-ru + version: pre1093-newstest2019-en-ru-json + sha256: 27b30a1ab0f8a6e4aa3025c3381a5e610a46c1d03140a9b50312601bcfe38c35 + source: + mode: manual + path: WMT19_en_ru/newstest2019_en_ru_task_eval.json + instructions: https://www.statmt.org/wmt19/translation-task.html + license: dataset-specific +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml b/tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml new file mode 100644 index 0000000000..3c18ae0d4f --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: wmt14-en-de-t5 + version: pre1093-wmt14-en-de-t5-json + sha256: 7851c9e2ed3627bbc8b0420415eeb2360422418a8e31e8dd5726b666c450479a + source: + mode: manual + path: WMT14_de_en/wmt14_en_de_t5_task_eval.json + instructions: https://www.statmt.org/wmt14/translation-task.html + license: dataset-specific +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py index cd95713538..a8251f88fa 100644 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ b/tools/benchmark_qualification/references/hf_text_generation.py @@ -10,7 +10,7 @@ import json import os from pathlib import Path -from typing import Any, Mapping +from typing import Any, Mapping, Sequence def main() -> int: @@ -21,7 +21,7 @@ def main() -> int: request = json.loads(arguments.request.read_text(encoding="utf-8")) import torch - from transformers import AutoModelForCausalLM, AutoTokenizer + from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer if not torch.cuda.is_available(): raise RuntimeError("the HF Accuracy reference requires CUDA") @@ -38,13 +38,16 @@ def main() -> int: tokenizer = AutoTokenizer.from_pretrained(str(request["model"]), **tokenizer_options) if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token - model = AutoModelForCausalLM.from_pretrained( + task = str(request.get("task", "causal-lm")) + model_type = _model_class(task, AutoModelForCausalLM, AutoModelForSeq2SeqLM) + model = model_type.from_pretrained( str(request["model"]), **_precision_load_options(dtypes[precision]), **model_options ).eval() model.to("cuda") generation = request.get("generation", {}) if not isinstance(generation, Mapping): raise ValueError("generation must be an object") + translation = _translation_controls(tokenizer, generation) results = [] with torch.inference_mode(): for sample in request["samples"]: @@ -63,6 +66,7 @@ def main() -> int: "pad_token_id": tokenizer.eos_token_id, "eos_token_id": tokenizer.eos_token_id, "repetition_penalty": float(generation.get("repetition_penalty", 1.0)), + **translation, } if options["do_sample"]: options.update( @@ -74,8 +78,16 @@ def main() -> int: if seed >= 0: torch.manual_seed(seed) generated = model.generate(**encoded, **options) - prompt_length = int(encoded["input_ids"].shape[-1]) + prompt_length = 0 if task == "seq2seq-lm" else int(encoded["input_ids"].shape[-1]) token_ids = generated[0, prompt_length:].detach().cpu().tolist() + if task == "seq2seq-lm": + generation_config = getattr(model, "generation_config", model.config) + token_ids = _normalize_seq2seq_tokens( + token_ids, + getattr(generation_config, "decoder_start_token_id", None), + getattr(generation_config, "eos_token_id", None), + str(request.get("output_token_policy", "new-tokens")), + ) results.append( { "sample_id": str(sample["sample_id"]), @@ -88,7 +100,9 @@ def main() -> int: "schema_version": "trtmc.accuracy-reference/v1", "model": request["model"], "revision": getattr(model.config, "_commit_hash", None) or request.get("revision"), + "task": task, "precision": precision, + "output_token_policy": str(request.get("output_token_policy", "new-tokens")), "samples": results, } arguments.output.write_text( @@ -116,6 +130,95 @@ def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: return options +def _model_class(task: str, causal: Any, seq2seq: Any) -> Any: + if task == "causal-lm": + return causal + if task == "seq2seq-lm": + return seq2seq + raise ValueError(f"unsupported reference task {task!r}") + + +def _translation_controls(tokenizer: Any, request: Mapping[str, Any]) -> dict[str, int]: + source = request.get("source_language") + target = request.get("target_language") + for name, value in (("source_language", source), ("target_language", target)): + if value is not None and (not isinstance(value, str) or not value): + raise ValueError(f"generation.{name} must be a nonempty language identifier") + + def token_id(language: str) -> int: + lookup = getattr(tokenizer, "get_lang_id", None) + if lookup is None: + lookup = tokenizer.convert_tokens_to_ids + value = lookup(language) + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value == getattr(tokenizer, "unk_token_id", None) + ): + raise ValueError(f"tokenizer does not recognize language {language!r}") + return value + + def explicit_id(name: str) -> int | None: + value = request.get(name) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value < -1: + raise ValueError(f"generation.{name} must be a nonnegative integer or -1") + return None if value == -1 else value + + source_id = explicit_id("source_language_token_id") + target_id = explicit_id("forced_bos_token_id") + if source_id is not None: + if source is not None and token_id(source) != source_id: + raise ValueError("source language disagrees with source_language_token_id") + source = tokenizer.convert_ids_to_tokens(source_id) + if source is not None: + if hasattr(tokenizer, "src_lang"): + token_id(source) + tokenizer.src_lang = source + elif source != getattr(tokenizer, "source_lang", None): + raise ValueError("reference tokenizer does not support the requested source language") + if target is not None: + if hasattr(tokenizer, "src_lang"): + resolved = token_id(target) + if target_id is not None and target_id != resolved: + raise ValueError("target language disagrees with forced_bos_token_id") + target_id = resolved + elif target != getattr(tokenizer, "target_lang", None): + raise ValueError("reference tokenizer does not support the requested target language") + return {} if target_id is None else {"forced_bos_token_id": target_id} + + +def _normalize_seq2seq_tokens( + token_ids: Sequence[int], + decoder_start_token_id: int | None, + eos_token_id: int | Sequence[int] | None, + policy: str, +) -> list[int]: + if policy not in {"new-tokens", "strip-start", "strip-start-and-eos"}: + raise ValueError(f"unsupported output token policy {policy!r}") + normalized = [int(value) for value in token_ids] + if policy == "new-tokens": + return normalized + if ( + normalized + and decoder_start_token_id is not None + and normalized[0] == int(decoder_start_token_id) + ): + normalized.pop(0) + if policy != "strip-start-and-eos" or not normalized or eos_token_id is None: + return normalized + eos_ids = ( + {int(eos_token_id)} + if isinstance(eos_token_id, int) + else {int(value) for value in eos_token_id} + ) + if normalized[-1] in eos_ids: + normalized.pop() + return normalized + + def _precision_load_options(dtype: Any) -> dict[str, Any]: return {"torch_dtype": dtype} diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 244cec2f57..5409caddeb 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -514,11 +514,15 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/__init__.py", "tools/benchmark_qualification/accuracy.py", "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", + "tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml", + "tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", + "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", + "tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml", "tools/benchmark_qualification/catalog.py", "tools/benchmark_qualification/datasets.py", "tools/benchmark_qualification/performance.py", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 73ce4b6801..c89cadf321 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -14,7 +14,12 @@ from apps.benchmark.performance.baselines.timing_contracts import timing_contract from tools.benchmark_qualification import accuracy as qualification_accuracy -from tools.benchmark_qualification.catalog import discover, load_benchmark, select +from tools.benchmark_qualification.catalog import ( + QualificationCase, + discover, + load_benchmark, + select, +) from tools.benchmark_qualification.datasets import Dataset, resolve_dataset from tools.benchmark_qualification.references import hf_text_generation from tools.benchmark_qualification.runtime import ( @@ -75,15 +80,19 @@ def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> N "glm-4-9b": 20, "granite-3.1-2b": 20, "internlm2-1.8b": 20, + "marian-en-ru": 20, "minitron-4b-depth": 20, "minitron-4b-width": 20, "mistral-7b": 10, "nemotron-hindi-4b": 20, + "nllb-200-distilled-600m": 20, "olmo2-1b": 8, "phi3-mini": 10, "qwen3-0.6b-fp16": 10, "stablelm2-1.6b": 22, "starcoder2-3b": 20, + "riva-translate-4b": 20, + "t5-small": 20, } for model, tokens in expected.items(): @@ -205,6 +214,175 @@ def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) } +def test_accuracy_forwards_seq2seq_reference_contract_and_nested_dataset_input( + tmp_path: Path, monkeypatch +) -> None: + dataset_path = tmp_path / "translation.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "sample_id": "translation-0", + "inputs": {"prompt": "The house is wonderful."}, + } + ] + } + ), + encoding="utf-8", + ) + case = QualificationCase( + kind="accuracy", + model="translation-model", + family="translation", + name="translation-parity", + benchmark="translation_parity", + candidate={ + "family": "translation", + "checkpoint": "org/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "prompt_token_limit": 96, + "reference": { + "precision": "fp16", + "task": "seq2seq-lm", + "output_token_policy": "strip-start-and-eos", + }, + "request": { + "max_new_tokens": 128, + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + }, + "gate": {"min_pass_rate": 1.0, "allowed_failures": 0}, + }, + source=tmp_path / "translation.yaml", + reference_requirements=None, + ) + dataset = Dataset("translation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured_reference: dict[str, object] = {} + captured_candidate: list[dict[str, object]] = [] + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured_reference.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "translation-0", + "prompt": "The house is wonderful.", + "token_ids": [4, 5], + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def candidate(_case, _context, _output, _operation, requests): + captured_candidate.extend(requests) + return ([{"token_ids": [4, 5]}], tmp_path / "translation.bundle") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "exact_token_ids"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr(qualification_accuracy, "_candidate_outputs", candidate) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert captured_reference["task"] == "seq2seq-lm" + assert captured_reference["output_token_policy"] == "strip-start-and-eos" + assert captured_reference["samples"] == [ + {"sample_id": "translation-0", "prompt": "The house is wonderful."} + ] + assert captured_candidate[0]["request"]["forced_bos_token_id"] == 256057 + + +def test_hf_accuracy_reference_selects_the_seq2seq_model_class() -> None: + causal = object() + seq2seq = object() + + assert hf_text_generation._model_class("causal-lm", causal, seq2seq) is causal + assert hf_text_generation._model_class("seq2seq-lm", causal, seq2seq) is seq2seq + with pytest.raises(ValueError, match="unsupported reference task"): + hf_text_generation._model_class("encoder", causal, seq2seq) + + +def test_hf_accuracy_reference_applies_explicit_translation_languages() -> None: + class Tokenizer: + src_lang = None + unk_token_id = 0 + + @staticmethod + def convert_tokens_to_ids(value: str) -> int: + return {"eng_Latn": 256047, "fra_Latn": 256057}.get(value, 0) + + @staticmethod + def convert_ids_to_tokens(value: int) -> str: + return {256047: "eng_Latn", 256057: "fra_Latn"}[value] + + tokenizer = Tokenizer() + controls = hf_text_generation._translation_controls( + tokenizer, + { + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + }, + ) + + assert tokenizer.src_lang == "eng_Latn" + assert controls == {"forced_bos_token_id": 256057} + + +def test_hf_accuracy_reference_normalizes_seq2seq_control_tokens() -> None: + assert hf_text_generation._normalize_seq2seq_tokens( + [2, 256057, 1034, 248075, 2], + decoder_start_token_id=2, + eos_token_id=2, + policy="strip-start-and-eos", + ) == [256057, 1034, 248075] + assert hf_text_generation._normalize_seq2seq_tokens( + [0, 17, 1], + decoder_start_token_id=0, + eos_token_id=1, + policy="strip-start", + ) == [17, 1] + + def test_shared_definitions_own_dataset_and_metric_not_models() -> None: for case in discover(REPOSITORY): definition = load_benchmark(REPOSITORY, case) From 658534d5e993537a4040999a8d8fb209cc19a89b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 17:10:48 +0000 Subject: [PATCH 016/100] feat(benchmark): restore encoder qualification Add family-local Accuracy and Performance cases for encoder and embedding models, with shared model-agnostic STS parity mechanics and explicit vector output from the legacy task worker. Also support the generic Transformers tokenizer used by NLLB while preserving explicit source-language validation discovered during hardware qualification. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 6 +- .../performance/baselines/hf_transformers.py | 48 ++- .../native/test_benchmark_worker_e2e.cpp | 29 ++ .../trtmc_benchmark/tests/test_perf_matrix.py | 8 +- core/runtime/tests/fake_family.cpp | 22 +- .../albert/tests/benchmark/albert-base.yaml | 41 +++ .../tests/benchmark/all-minilm-l6-v2.yaml | 41 +++ .../tests/benchmark/bert-base-uncased.yaml | 41 +++ .../deberta/tests/benchmark/deberta-base.yaml | 41 +++ .../benchmark/distilbert-base-uncased.yaml | 41 +++ .../dpr/tests/benchmark/dpr-ctx-encoder.yaml | 42 +++ .../tests/benchmark/modernbert-base.yaml | 41 +++ .../tests/benchmark/all-mpnet-base-v2.yaml | 41 +++ .../roberta/tests/benchmark/roberta-base.yaml | 41 +++ tools/benchmark_qualification/accuracy.py | 338 ++++++++++++++++++ .../benchmarks/embedding_performance.yaml | 15 + .../benchmarks/encoder_performance.yaml | 15 + .../stsbenchmark_embedding_parity.yaml | 20 ++ .../references/hf_encoder.py | 138 +++++++ .../references/hf_text_generation.py | 48 ++- tools/tests/test_architecture.py | 4 + tools/tests/test_model_benchmark.py | 312 +++++++++++++++- 22 files changed, 1354 insertions(+), 19 deletions(-) create mode 100644 families/albert/tests/benchmark/albert-base.yaml create mode 100644 families/bert/tests/benchmark/all-minilm-l6-v2.yaml create mode 100644 families/bert/tests/benchmark/bert-base-uncased.yaml create mode 100644 families/deberta/tests/benchmark/deberta-base.yaml create mode 100644 families/distilbert/tests/benchmark/distilbert-base-uncased.yaml create mode 100644 families/dpr/tests/benchmark/dpr-ctx-encoder.yaml create mode 100644 families/modernbert/tests/benchmark/modernbert-base.yaml create mode 100644 families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml create mode 100644 families/roberta/tests/benchmark/roberta-base.yaml create mode 100644 tools/benchmark_qualification/benchmarks/embedding_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/encoder_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml create mode 100644 tools/benchmark_qualification/references/hf_encoder.py diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 9af73aa010..c4a552f524 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -613,10 +613,12 @@ Json run_embedding(trtmc::ITask& task, const Json& request, const Timing& timing auto& interface = require_interface(task, "IEncoding"); invoke = [&interface, prompt]() { return interface.encode(prompt); }; } - return measure(timing, invoke, [](const trtmc::EmbeddingResult& result) { + return measure(timing, invoke, [pooled](const trtmc::EmbeddingResult& result) { return Json{{"embedding_vectors", 1}, {"embedding_elements", result.data.size()}, - {"dim", result.dim}}; + {"dim", result.dim}, + {"values", result.data}, + {"feature_kind", pooled ? "pooled" : "token"}}; }); } diff --git a/apps/benchmark/performance/baselines/hf_transformers.py b/apps/benchmark/performance/baselines/hf_transformers.py index 47d37f097b..41fa8eb397 100644 --- a/apps/benchmark/performance/baselines/hf_transformers.py +++ b/apps/benchmark/performance/baselines/hf_transformers.py @@ -179,7 +179,9 @@ def _batch_prompt(request: Mapping[str, Any]) -> list[str]: return [prompt] * batch_size -def _translation_controls(tokenizer: Any, request: Mapping[str, Any]) -> dict[str, int]: +def _translation_controls( + tokenizer: Any, request: Mapping[str, Any] +) -> tuple[dict[str, int], int | None]: """Use tokenizer language APIs; fixed-pair models validate their declared pair.""" source = request.get("source_language") target = request.get("target_language") @@ -211,6 +213,7 @@ def explicit_id(name: str) -> int | None: source_id = explicit_id("source_language_token_id") target_id = explicit_id("forced_bos_token_id") + manual_source_id = None if source_id is not None: if source is not None and token_id(source) != source_id: raise ValueError("source language disagrees with source_language_token_id") @@ -219,7 +222,11 @@ def explicit_id(name: str) -> int | None: if hasattr(tokenizer, "src_lang"): token_id(source) tokenizer.src_lang = source - elif source != getattr(tokenizer, "source_lang", None): + elif source == getattr(tokenizer, "source_lang", None): + pass + elif source_id is not None and token_id(source) == source_id: + manual_source_id = source_id + else: raise ValueError("reference tokenizer does not support the requested source language") if target is not None: if hasattr(tokenizer, "src_lang"): @@ -227,9 +234,39 @@ def explicit_id(name: str) -> int | None: if target_id is not None and target_id != resolved: raise ValueError("target language disagrees with forced_bos_token_id") target_id = resolved - elif target != getattr(tokenizer, "target_lang", None): + elif target == getattr(tokenizer, "target_lang", None): + pass + elif target_id is not None and token_id(target) == target_id: + pass + else: raise ValueError("reference tokenizer does not support the requested target language") - return {} if target_id is None else {"forced_bos_token_id": target_id} + controls = {} if target_id is None else {"forced_bos_token_id": target_id} + return controls, manual_source_id + + +def _apply_source_language( + encoded: Mapping[str, Any], source_token_id: int | None, tokenizer: Any +) -> None: + if source_token_id is None: + return + input_ids = encoded.get("input_ids") + if input_ids is None or getattr(input_ids, "ndim", None) != 2: + raise ValueError("tokenized translation input must contain rank-2 input_ids") + attention_mask = encoded.get("attention_mask") + for row in range(int(input_ids.shape[0])): + if attention_mask is None: + index = int(input_ids.shape[1]) - 1 + else: + positions = attention_mask[row].nonzero(as_tuple=False) + if int(positions.numel()) == 0: + raise ValueError("tokenized translation input cannot be empty") + index = int(positions[-1].item()) + current = int(input_ids[row, index].item()) + if current == source_token_id: + continue + if current != getattr(tokenizer, "unk_token_id", None): + raise ValueError("generic translation tokenizer did not emit a language placeholder") + input_ids[row, index] = source_token_id def _encoder_call( @@ -324,7 +361,7 @@ def encode_prompt() -> Any: "use_cache": True, "pad_token_id": tokenizer.pad_token_id, } - translation = _translation_controls(tokenizer, request) + translation, source_language_token_id = _translation_controls(tokenizer, request) if generation_method == "ar-generate" and translation: raise ValueError("ar-generate reference does not accept translation controls") generation.update(translation) @@ -345,6 +382,7 @@ def encode_prompt() -> Any: def invoke() -> dict[str, Any]: encoded = encode_prompt() + _apply_source_language(encoded, source_language_token_id, tokenizer) gpu_inputs = {name: value.to("cuda") for name, value in encoded.items()} with ( torch.inference_mode(), diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index 344142202c..fe197f3854 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace { @@ -130,6 +131,34 @@ int main(int argc, char** argv) { check(summary.at("forecast_elements") == 3, "summary values returned"); check(summary.at("shape") == Json::array({1, 3}), "summary shape returned"); + for (const auto& [task, operation, expected_values, feature_kind] : { + std::tuple{"encoding", "encode", Json::array({5.0, 1.0, 2.0, 3.0}), "token"}, + std::tuple{"embedding", "embed", Json::array({5.0, 4.0}), "pooled"}, + }) { + const auto vector_bundle = + runtime_root / (std::string("benchmark_fake_") + task + ".bundle"); + write_bundle(vector_bundle, task); + Json vector_request = request; + vector_request["case_name"] = std::string("fake-") + task; + vector_request["bundle"] = vector_bundle.string(); + vector_request["operation"] = operation; + vector_request["request"] = {{"prompt", "Hello"}}; + { + std::ofstream request_file(request_path); + request_file << vector_request << '\n'; + } + check(std::system(command.c_str()) == 0, "encoder worker process completed"); + std::ifstream vector_output_file(output_path); + Json vector_result; + vector_output_file >> vector_result; + const auto& vector_summary = vector_result.at("output_summary"); + check(vector_summary.at("values") == expected_values, + "legacy encoder output retains values for Accuracy comparison"); + check(vector_summary.at("feature_kind") == feature_kind, + "legacy encoder output identifies token or pooled semantics"); + std::filesystem::remove(vector_bundle); + } + Json sdk = request; sdk["runtime_root"] = argv[3]; sdk["operation"] = "generate"; diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 17888977e7..1a71e33fbc 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -132,15 +132,15 @@ def convert_ids_to_tokens(self, token): return {10: "eng_Latn", 11: "fra_Latn"}[token] tokenizer = Tokenizer() - assert hf_transformers._translation_controls(tokenizer, {}) == {} + assert hf_transformers._translation_controls(tokenizer, {}) == ({}, None) assert tokenizer.src_lang == "default" assert hf_transformers._translation_controls( tokenizer, {"source_language": "eng_Latn", "target_language": "fra_Latn"} - ) == {"forced_bos_token_id": 11} + ) == ({"forced_bos_token_id": 11}, None) assert tokenizer.src_lang == "eng_Latn" assert hf_transformers._translation_controls( tokenizer, {"source_language_token_id": 10, "forced_bos_token_id": 0} - ) == {"forced_bos_token_id": 0} + ) == ({"forced_bos_token_id": 0}, None) with pytest.raises(ValueError, match="disagrees"): hf_transformers._translation_controls( tokenizer, {"target_language": "fra_Latn", "forced_bos_token_id": 10} @@ -152,7 +152,7 @@ def convert_ids_to_tokens(self, token): hf_transformers._translation_controls( fixed, {"source_language": "en", "target_language": "ru"} ) - == {} + == ({}, None) ) with pytest.raises(ValueError, match="target language"): hf_transformers._translation_controls(fixed, {"target_language": "de"}) diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 289084c219..f887e4fb3d 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -42,6 +42,20 @@ class FakeForecast final : public trtmc::ITimeSeriesForecast { std::uint64_t kv_cache_size_bytes_; }; +class FakeEncoding final : public trtmc::IEncoding { + public: + trtmc::EmbeddingResult encode(const std::string& text) override { + return {{static_cast(text.size()), 1.0F, 2.0F, 3.0F}, 2}; + } +}; + +class FakeEmbedding final : public trtmc::IEmbedding { + public: + trtmc::EmbeddingResult embed(const std::string& text) override { + return {{static_cast(text.size()), 4.0F}, 2}; + } +}; + } // namespace extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { @@ -55,5 +69,11 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context const auto plan = context.reader.read_section("engine.plan"); if (std::string(plan.begin(), plan.end()) != "PLAN") throw std::runtime_error("unexpected engine plan"); - return new FakeForecast(context.backend, context.kv_cache_size_bytes); + if (context.reader.info().task == trtmc::IEncoding::kTask) + return new FakeEncoding(); + if (context.reader.info().task == trtmc::IEmbedding::kTask) + return new FakeEmbedding(); + if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) + return new FakeForecast(context.backend, context.kv_cache_size_bytes); + throw std::runtime_error("unsupported fake task"); } diff --git a/families/albert/tests/benchmark/albert-base.yaml b/families/albert/tests/benchmark/albert-base.yaml new file mode 100644 index 0000000000..62ad5286dc --- /dev/null +++ b/families/albert/tests/benchmark/albert-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: albert-base +candidate: + family: albert + checkpoint: albert/albert-base-v2 + task: encoding + precision: fp16 + bundle: albert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.99 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/all-minilm-l6-v2.yaml b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml new file mode 100644 index 0000000000..e2ea697916 --- /dev/null +++ b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: all-minilm-l6-v2 +candidate: + family: bert + checkpoint: sentence-transformers/all-MiniLM-L6-v2 + task: embedding + precision: fp16 + bundle: all-minilm-l6-v2.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: embed + benchmark: embedding_performance + operation: embed + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-embedding + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/bert-base-uncased.yaml b/families/bert/tests/benchmark/bert-base-uncased.yaml new file mode 100644 index 0000000000..2106df7abf --- /dev/null +++ b/families/bert/tests/benchmark/bert-base-uncased.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: bert-base-uncased +candidate: + family: bert + checkpoint: google-bert/bert-base-uncased + task: encoding + precision: fp16 + bundle: bert-base-uncased.bundle + build: + max_sequence_length: 128 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/deberta/tests/benchmark/deberta-base.yaml b/families/deberta/tests/benchmark/deberta-base.yaml new file mode 100644 index 0000000000..352cd94941 --- /dev/null +++ b/families/deberta/tests/benchmark/deberta-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deberta-base +candidate: + family: deberta + checkpoint: microsoft/deberta-base + task: encoding + precision: fp16 + bundle: deberta-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml b/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml new file mode 100644 index 0000000000..0358cd2e6b --- /dev/null +++ b/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: distilbert-base-uncased +candidate: + family: distilbert + checkpoint: distilbert/distilbert-base-uncased + task: encoding + precision: fp16 + bundle: distilbert-base-uncased.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml new file mode 100644 index 0000000000..0df5442ce2 --- /dev/null +++ b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dpr-ctx-encoder +candidate: + family: dpr + checkpoint: facebook/dpr-ctx_encoder-single-nq-base + task: encoding + precision: fp16 + bundle: dpr-ctx-encoder.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + model_class: dpr-context-encoder + gate: + min_vector_cosine: 0.99 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/modernbert/tests/benchmark/modernbert-base.yaml b/families/modernbert/tests/benchmark/modernbert-base.yaml new file mode 100644 index 0000000000..08cce686c6 --- /dev/null +++ b/families/modernbert/tests/benchmark/modernbert-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: modernbert-base +candidate: + family: modernbert + checkpoint: answerdotai/ModernBERT-base + task: encoding + precision: fp32 + bundle: modernbert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.995 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml b/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml new file mode 100644 index 0000000000..e460796826 --- /dev/null +++ b/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: all-mpnet-base-v2 +candidate: + family: mpnet + checkpoint: sentence-transformers/all-mpnet-base-v2 + task: encoding + precision: fp16 + bundle: all-mpnet-base-v2.bundle + build: + max_sequence_length: 384 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/roberta/tests/benchmark/roberta-base.yaml b/families/roberta/tests/benchmark/roberta-base.yaml new file mode 100644 index 0000000000..54d104417d --- /dev/null +++ b/families/roberta/tests/benchmark/roberta-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: roberta-base +candidate: + family: roberta + checkpoint: FacebookAI/roberta-base + task: encoding + precision: fp16 + bundle: roberta-base.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 7a7c382ffe..cacc2b4878 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -34,6 +34,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, metric_name = metric.get("name") if isinstance(metric, Mapping) else None if metric_name == "exact_token_ids": result = _text_generation_parity(case, context, dataset, output) + elif metric_name == "embedding_vector_parity": + result = _encoder_embedding_parity(case, context, dataset, output) elif metric_name == "forecast_tensor_parity": result = _etth1(case, context, definition, dataset, output) else: @@ -169,6 +171,342 @@ def _text_generation_parity( } +def _encoder_embedding_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + prompt_prefix = configured.get("prompt_prefix", "") + if not isinstance(prompt_prefix, str): + raise QualificationError("accuracy.prompt_prefix must be a string") + samples = _sts_samples(dataset.path, sample_limit, prompt_prefix) + reference = configured.get("reference", {}) + gates = configured.get("gate", {}) + if not isinstance(reference, Mapping) or not isinstance(gates, Mapping): + raise QualificationError("encoder Accuracy reference and gate must be objects") + task = str(case.candidate["task"]) + try: + mode, operation = { + "encoding": ("cls", "encode"), + "embedding": ("embedding", "embed"), + }[task] + except KeyError as error: + raise QualificationError( + f"embedding vector parity does not support candidate task {task!r}" + ) from error + model_class = str(reference.get("model_class", "auto")) + if model_class not in {"auto", "dpr-context-encoder"}: + raise QualificationError(f"unsupported encoder reference model class {model_class!r}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "trust_remote_code": bool(case.candidate.get("trust_remote_code", False)), + "precision": str(reference.get("precision", "fp32")), + "mode": mode, + "model_class": model_class, + "max_length": int(case.candidate.get("build", {}).get("max_sequence_length", 512)), + "samples": samples, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + output.mkdir(parents=True, exist_ok=True) + _json(request_path, reference_request) + runner = context.repository / "tools/benchmark_qualification/references/hf_encoder.py" + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"HF encoder Accuracy reference failed; see {output}") + reference_samples = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(reference_samples, list) or len(reference_samples) != len(samples): + raise QualificationError("HF encoder Accuracy reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"prompt": sample["prompt"]}, + } + for sample in samples + ] + candidate, bundle = _candidate_outputs( + case, context, output, operation, candidate_requests + ) + compared = _compare_encoder_embeddings(reference_samples, candidate, gates) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": compared["status"], + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": compared["metrics"], + "gate": compared["gate"], + "samples": compared["samples"], + "pairs": compared["pairs"], + } + + +def _sts_samples(path: Path, count: int, prefix: str) -> list[dict[str, Any]]: + pairs: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as stream: + for dataset_index, line in enumerate(stream): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise QualificationError( + f"STSBenchmark row {dataset_index} is not valid JSON" + ) from error + if not isinstance(row, Mapping): + raise QualificationError(f"STSBenchmark row {dataset_index} must be an object") + sentence1 = row.get("sentence1") + sentence2 = row.get("sentence2") + if not all(isinstance(value, str) and value.strip() for value in (sentence1, sentence2)): + raise QualificationError( + f"STSBenchmark row {dataset_index} must contain two sentences" + ) + try: + score = float(row["score"]) + except (KeyError, TypeError, ValueError) as error: + raise QualificationError( + f"STSBenchmark row {dataset_index} has an invalid score" + ) from error + if not math.isfinite(score): + raise QualificationError( + f"STSBenchmark row {dataset_index} has a non-finite score" + ) + pair_id = f"stsbenchmark-{dataset_index:06d}" + for suffix, side, sentence in ( + ("a", "sentence1", sentence1), + ("b", "sentence2", sentence2), + ): + pairs.append( + { + "sample_id": f"{pair_id}-{suffix}", + "pair_id": pair_id, + "pair_side": side, + "score": score, + "prompt": prefix + str(sentence).strip(), + } + ) + if len(pairs) == count * 2: + break + if len(pairs) != count * 2: + raise QualificationError( + f"STSBenchmark contains {len(pairs) // 2} usable pairs; {count} required" + ) + return pairs + + +def _compare_encoder_embeddings( + reference: Sequence[Mapping[str, Any]], + candidate: Sequence[Mapping[str, Any]], + gates: Mapping[str, Any], +) -> dict[str, Any]: + if len(reference) != len(candidate) or not reference: + raise QualificationError("encoder reference and candidate sample counts differ") + minimum_cosine = float(gates.get("min_vector_cosine", 0.999)) + minimum_rate = float(gates.get("min_vector_pass_rate", 1.0)) + maximum_pair_delta = float(gates.get("max_pair_cosine_abs_delta", 0.02)) + if not -1.0 <= minimum_cosine <= 1.0: + raise QualificationError("min_vector_cosine must be in [-1, 1]") + if not 0.0 <= minimum_rate <= 1.0: + raise QualificationError("min_vector_pass_rate must be in [0, 1]") + if maximum_pair_delta < 0.0 or not math.isfinite(maximum_pair_delta): + raise QualificationError("max_pair_cosine_abs_delta must be finite and nonnegative") + + sample_rows: list[dict[str, Any]] = [] + vector_cosines: list[float] = [] + reference_pairs: dict[str, dict[str, tuple[Mapping[str, Any], list[float]]]] = {} + candidate_pairs: dict[str, dict[str, list[float]]] = {} + for index, (expected, actual) in enumerate(zip(reference, candidate, strict=True)): + expected_id = str(expected.get("sample_id", index)) + vector = expected.get("vector") + if not isinstance(vector, list): + raise QualificationError(f"HF encoder sample {expected_id!r} has no vector") + reference_vector = _finite_vector(vector, f"HF encoder sample {expected_id!r}") + candidate_vector = _candidate_vector(actual, expected_id) + cosine = _vector_cosine(reference_vector, candidate_vector) + pair_id = str(expected.get("pair_id", "")) + pair_side = str(expected.get("pair_side", "")) + if not pair_id or pair_side not in {"sentence1", "sentence2"}: + raise QualificationError(f"HF encoder sample {expected_id!r} has invalid pair metadata") + reference_pairs.setdefault(pair_id, {})[pair_side] = (expected, reference_vector) + candidate_pairs.setdefault(pair_id, {})[pair_side] = candidate_vector + vector_cosines.append(cosine) + sample_rows.append( + { + "sample_id": expected_id, + "pair_id": pair_id, + "pair_side": pair_side, + "vector_dim": len(reference_vector), + "vector_cosine": cosine, + "passed": cosine >= minimum_cosine, + } + ) + + pair_rows: list[dict[str, Any]] = [] + pair_deltas: list[float] = [] + scores: list[float] = [] + reference_similarities: list[float] = [] + candidate_similarities: list[float] = [] + for pair_id, expected in reference_pairs.items(): + actual = candidate_pairs.get(pair_id, {}) + if set(expected) != {"sentence1", "sentence2"} or set(actual) != { + "sentence1", + "sentence2", + }: + raise QualificationError(f"encoder pair {pair_id!r} is incomplete") + reference_similarity = _vector_cosine( + expected["sentence1"][1], expected["sentence2"][1] + ) + candidate_similarity = _vector_cosine(actual["sentence1"], actual["sentence2"]) + delta = abs(candidate_similarity - reference_similarity) + score = float(expected["sentence1"][0]["score"]) + pair_deltas.append(delta) + scores.append(score) + reference_similarities.append(reference_similarity) + candidate_similarities.append(candidate_similarity) + pair_rows.append( + { + "pair_id": pair_id, + "score": score, + "hf_cosine": reference_similarity, + "candidate_cosine": candidate_similarity, + "cosine_abs_delta": delta, + "passed": delta <= maximum_pair_delta, + } + ) + + passed_vectors = sum(value >= minimum_cosine for value in vector_cosines) + pass_rate = passed_vectors / len(vector_cosines) + max_pair_delta = max(pair_deltas) + status = ( + "passed" + if pass_rate >= minimum_rate and max_pair_delta <= maximum_pair_delta + else "failed" + ) + return { + "status": status, + "metrics": { + "samples": len(vector_cosines), + "pairs": len(pair_rows), + "passed_vectors": passed_vectors, + "vector_pass_rate": pass_rate, + "mean_vector_cosine": math.fsum(vector_cosines) / len(vector_cosines), + "min_vector_cosine": min(vector_cosines), + "mean_pair_cosine_abs_delta": math.fsum(pair_deltas) / len(pair_deltas), + "max_pair_cosine_abs_delta": max_pair_delta, + "hf_sts_spearman": _spearman(scores, reference_similarities), + "candidate_sts_spearman": _spearman(scores, candidate_similarities), + }, + "gate": { + "min_vector_cosine": minimum_cosine, + "min_vector_pass_rate": minimum_rate, + "max_pair_cosine_abs_delta": maximum_pair_delta, + }, + "samples": sample_rows, + "pairs": pair_rows, + } + + +def _candidate_vector(summary: Mapping[str, Any], sample_id: str) -> list[float]: + values = summary.get("values") + if not isinstance(values, list): + raise QualificationError(f"TRTMC encoder sample {sample_id!r} has no vector values") + vector = _finite_vector(values, f"TRTMC encoder sample {sample_id!r}") + dim = summary.get("dim") + if isinstance(dim, bool) or not isinstance(dim, int) or dim < 1: + dim = len(vector) + if summary.get("feature_kind") == "token": + if len(vector) < dim: + raise QualificationError(f"TRTMC encoder sample {sample_id!r} is shorter than dim") + vector = vector[:dim] + elif len(vector) != dim: + raise QualificationError( + f"TRTMC encoder sample {sample_id!r} has {len(vector)} values for dim {dim}" + ) + return vector + + +def _finite_vector(values: Sequence[Any], label: str) -> list[float]: + try: + vector = [float(value) for value in values] + except (TypeError, ValueError) as error: + raise QualificationError(f"{label} contains a non-numeric value") from error + if not vector or any(not math.isfinite(value) for value in vector): + raise QualificationError(f"{label} must be non-empty and finite") + return vector + + +def _vector_cosine(left: Sequence[float], right: Sequence[float]) -> float: + if len(left) != len(right) or not left: + raise QualificationError( + f"encoder vector dimensions differ: {len(left)} != {len(right)}" + ) + left_norm = math.sqrt(math.fsum(value * value for value in left)) + right_norm = math.sqrt(math.fsum(value * value for value in right)) + if left_norm <= 0.0 or right_norm <= 0.0: + raise QualificationError("encoder vector must have a nonzero norm") + cosine = math.fsum(a * b for a, b in zip(left, right, strict=True)) / ( + left_norm * right_norm + ) + return max(-1.0, min(1.0, cosine)) + + +def _spearman(left: Sequence[float], right: Sequence[float]) -> float | None: + if len(left) != len(right) or len(left) < 2: + return None + return _pearson(_ranks(left), _ranks(right)) + + +def _ranks(values: Sequence[float]) -> list[float]: + indexed = sorted(enumerate(values), key=lambda item: item[1]) + ranks = [0.0] * len(values) + start = 0 + while start < len(indexed): + end = start + 1 + while end < len(indexed) and indexed[end][1] == indexed[start][1]: + end += 1 + rank = (start + end - 1) / 2.0 + for position in range(start, end): + ranks[indexed[position][0]] = rank + start = end + return ranks + + +def _pearson(left: Sequence[float], right: Sequence[float]) -> float | None: + left_mean = math.fsum(left) / len(left) + right_mean = math.fsum(right) / len(right) + centered_left = [value - left_mean for value in left] + centered_right = [value - right_mean for value in right] + denominator = math.sqrt( + math.fsum(value * value for value in centered_left) + * math.fsum(value * value for value in centered_right) + ) + if denominator <= 0.0: + return None + return math.fsum( + a * b for a, b in zip(centered_left, centered_right, strict=True) + ) / denominator + + def _etth1( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/embedding_performance.yaml b/tools/benchmark_qualification/benchmarks/embedding_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/embedding_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/encoder_performance.yaml b/tools/benchmark_qualification/benchmarks/encoder_performance.yaml new file mode 100644 index 0000000000..70c8984fbf --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/encoder_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: public_operation_call_wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml b/tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml new file mode 100644 index 0000000000..1e6c4e3cf2 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: stsbenchmark-test + version: pre1093-mteb-stsbenchmark-test-jsonl + sha256: 7c8927eef1ac4e256cae9768afc76232f79c39d19cb05d16bef92e11cecbe5bb + source: + mode: manual + path: STSBenchmark/stsbenchmark_test.jsonl + instructions: https://huggingface.co/datasets/mteb/stsbenchmark-sts + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: sts_pair_jsonl +metric: + name: embedding_vector_parity diff --git a/tools/benchmark_qualification/references/hf_encoder.py b/tools/benchmark_qualification/references/hf_encoder.py new file mode 100644 index 0000000000..44a56ded49 --- /dev/null +++ b/tools/benchmark_qualification/references/hf_encoder.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Produce Hugging Face encoder vectors for STS qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any, Mapping + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if not torch.cuda.is_available(): + raise RuntimeError("the HF encoder Accuracy reference requires CUDA") + precision = str(request.get("precision", "fp32")) + dtypes = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16} + if precision not in dtypes: + raise ValueError(f"unsupported reference precision {precision!r}") + mode = _vector_mode(str(request.get("mode", "cls"))) + model_class, tokenizer_class = _reference_classes( + transformers, str(request.get("model_class", "auto")) + ) + load_options = _model_load_options(request) + tokenizer_options = { + name: load_options[name] + for name in ("local_files_only", "revision", "trust_remote_code") + if name in load_options + } + tokenizer = tokenizer_class.from_pretrained(str(request["model"]), **tokenizer_options) + model = model_class.from_pretrained( + str(request["model"]), torch_dtype=dtypes[precision], **load_options + ).eval() + model.to("cuda") + max_length = int(request.get("max_length", 512)) + if max_length < 1: + raise ValueError("max_length must be positive") + + results = [] + with torch.inference_mode(): + for sample in request["samples"]: + encoded = tokenizer( + str(sample["prompt"]), + return_tensors="pt", + truncation=True, + max_length=max_length, + ) + encoded = {name: value.to("cuda") for name, value in encoded.items()} + outputs = model(**encoded, output_hidden_states=True) + hidden = getattr(outputs, "last_hidden_state", None) + if hidden is None: + hidden_states = getattr(outputs, "hidden_states", None) + if hidden_states: + hidden = hidden_states[-1] + elif isinstance(outputs, (tuple, list)) and outputs: + hidden = outputs[0] + if hidden is None or hidden.ndim != 3: + raise RuntimeError( + f"HF encoder output for {sample['sample_id']} has no rank-3 hidden state" + ) + if mode == "embedding": + attention_mask = encoded.get("attention_mask") + if attention_mask is None: + attention_mask = torch.ones(hidden.shape[:2], device=hidden.device) + mask = attention_mask.unsqueeze(-1).to(hidden.dtype) + vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) + vector = torch.nn.functional.normalize(vector, p=2, dim=-1)[0] + else: + vector = hidden[0, 0] + results.append( + { + "sample_id": str(sample["sample_id"]), + "pair_id": str(sample["pair_id"]), + "pair_side": str(sample["pair_side"]), + "score": float(sample["score"]), + "vector": [float(value) for value in vector.float().cpu().tolist()], + } + ) + output = { + "schema_version": "trtmc.encoder-reference/v1", + "model": request["model"], + "revision": getattr(model.config, "_commit_hash", None) or request.get("revision"), + "precision": precision, + "mode": mode, + "samples": results, + } + arguments.output.write_text( + json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +def _vector_mode(value: str) -> str: + if value not in {"cls", "embedding"}: + raise ValueError(f"unsupported encoder vector mode {value!r}") + return value + + +def _reference_classes(transformers_module: Any, value: str) -> tuple[Any, Any]: + if value == "auto": + return transformers_module.AutoModel, transformers_module.AutoTokenizer + if value == "dpr-context-encoder": + return ( + transformers_module.DPRContextEncoder, + transformers_module.DPRContextEncoderTokenizerFast, + ) + raise ValueError(f"unsupported encoder reference model class {value!r}") + + +def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: + options: dict[str, Any] = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + revision = request.get("revision") + if revision: + options["revision"] = revision + trust_remote_code = request.get("trust_remote_code") + if trust_remote_code is not None: + if not isinstance(trust_remote_code, bool): + raise ValueError("trust_remote_code must be a boolean") + options["trust_remote_code"] = trust_remote_code + return options + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py index a8251f88fa..92a92b7cff 100644 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ b/tools/benchmark_qualification/references/hf_text_generation.py @@ -47,7 +47,7 @@ def main() -> int: generation = request.get("generation", {}) if not isinstance(generation, Mapping): raise ValueError("generation must be an object") - translation = _translation_controls(tokenizer, generation) + translation, source_language_token_id = _translation_controls(tokenizer, generation) results = [] with torch.inference_mode(): for sample in request["samples"]: @@ -58,6 +58,7 @@ def main() -> int: str(request.get("truncation_side", "left")), ) encoded = tokenizer(prompt, return_tensors="pt") + _apply_source_language(encoded, source_language_token_id, tokenizer) encoded = {name: value.to("cuda") for name, value in encoded.items()} options: dict[str, Any] = { "max_new_tokens": int(generation.get("max_new_tokens", 64)), @@ -138,7 +139,9 @@ def _model_class(task: str, causal: Any, seq2seq: Any) -> Any: raise ValueError(f"unsupported reference task {task!r}") -def _translation_controls(tokenizer: Any, request: Mapping[str, Any]) -> dict[str, int]: +def _translation_controls( + tokenizer: Any, request: Mapping[str, Any] +) -> tuple[dict[str, int], int | None]: source = request.get("source_language") target = request.get("target_language") for name, value in (("source_language", source), ("target_language", target)): @@ -169,6 +172,7 @@ def explicit_id(name: str) -> int | None: source_id = explicit_id("source_language_token_id") target_id = explicit_id("forced_bos_token_id") + manual_source_id = None if source_id is not None: if source is not None and token_id(source) != source_id: raise ValueError("source language disagrees with source_language_token_id") @@ -177,7 +181,11 @@ def explicit_id(name: str) -> int | None: if hasattr(tokenizer, "src_lang"): token_id(source) tokenizer.src_lang = source - elif source != getattr(tokenizer, "source_lang", None): + elif source == getattr(tokenizer, "source_lang", None): + pass + elif source_id is not None and token_id(source) == source_id: + manual_source_id = source_id + else: raise ValueError("reference tokenizer does not support the requested source language") if target is not None: if hasattr(tokenizer, "src_lang"): @@ -185,9 +193,39 @@ def explicit_id(name: str) -> int | None: if target_id is not None and target_id != resolved: raise ValueError("target language disagrees with forced_bos_token_id") target_id = resolved - elif target != getattr(tokenizer, "target_lang", None): + elif target == getattr(tokenizer, "target_lang", None): + pass + elif target_id is not None and token_id(target) == target_id: + pass + else: raise ValueError("reference tokenizer does not support the requested target language") - return {} if target_id is None else {"forced_bos_token_id": target_id} + controls = {} if target_id is None else {"forced_bos_token_id": target_id} + return controls, manual_source_id + + +def _apply_source_language( + encoded: Mapping[str, Any], source_token_id: int | None, tokenizer: Any +) -> None: + if source_token_id is None: + return + input_ids = encoded.get("input_ids") + if input_ids is None or getattr(input_ids, "ndim", None) != 2: + raise ValueError("tokenized translation input must contain rank-2 input_ids") + attention_mask = encoded.get("attention_mask") + for row in range(int(input_ids.shape[0])): + if attention_mask is None: + index = int(input_ids.shape[1]) - 1 + else: + positions = attention_mask[row].nonzero(as_tuple=False) + if int(positions.numel()) == 0: + raise ValueError("tokenized translation input cannot be empty") + index = int(positions[-1].item()) + current = int(input_ids[row, index].item()) + if current == source_token_id: + continue + if current != getattr(tokenizer, "unk_token_id", None): + raise ValueError("generic translation tokenizer did not emit a language placeholder") + input_ids[row, index] = source_token_id def _normalize_seq2seq_tokens( diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 5409caddeb..493ca8bfd8 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -513,12 +513,15 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/test_impact.py", "tools/benchmark_qualification/__init__.py", "tools/benchmark_qualification/accuracy.py", + "tools/benchmark_qualification/benchmarks/embedding_performance.yaml", + "tools/benchmark_qualification/benchmarks/encoder_performance.yaml", "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", "tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml", "tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", + "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", @@ -526,6 +529,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/catalog.py", "tools/benchmark_qualification/datasets.py", "tools/benchmark_qualification/performance.py", + "tools/benchmark_qualification/references/hf_encoder.py", "tools/benchmark_qualification/references/hf_text_generation.py", "tools/benchmark_qualification/runtime.py", "tools/ci/__init__.py", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index c89cadf321..1ee83947a3 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -12,6 +12,7 @@ import pytest +from apps.benchmark.performance.baselines import hf_transformers from apps.benchmark.performance.baselines.timing_contracts import timing_contract from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification.catalog import ( @@ -21,7 +22,7 @@ select, ) from tools.benchmark_qualification.datasets import Dataset, resolve_dataset -from tools.benchmark_qualification.references import hf_text_generation +from tools.benchmark_qualification.references import hf_encoder, hf_text_generation from tools.benchmark_qualification.runtime import ( RuntimeContext, prepare_bundle, @@ -58,6 +59,77 @@ def test_one_model_file_owns_multiple_cases_without_testcase_indirection() -> No assert case.candidate["checkpoint"] == "openai-community/gpt2" +def test_bert_benchmark_uses_shared_sts_accuracy_and_encoder_performance() -> None: + cases = select(discover(REPOSITORY), ["bert-base-uncased"]) + + assert {case.kind for case in cases} == {"accuracy", "performance"} + accuracy = next(case for case in cases if case.kind == "accuracy") + performance = next(case for case in cases if case.kind == "performance") + assert accuracy.benchmark == "stsbenchmark_embedding_parity" + assert accuracy.values["samples"] == 50 + assert accuracy.values["gate"] == { + "min_vector_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "max_pair_cosine_abs_delta": 0.02, + } + assert load_benchmark(REPOSITORY, accuracy)["metric"]["name"] == ( + "embedding_vector_parity" + ) + assert performance.benchmark == "encoder_performance" + assert performance.values["operation"] == "encode" + assert performance.values["reference"]["mode"] == "torch-compile" + assert performance.values["reference"]["fallback"] == "hf-eager" + + +@pytest.mark.parametrize( + ("model", "family", "task", "precision", "minimum_cosine"), + [ + ("albert-base", "albert", "encoding", "fp16", 0.99), + ("all-minilm-l6-v2", "bert", "embedding", "fp16", 0.999), + ("all-mpnet-base-v2", "mpnet", "encoding", "fp16", 0.999), + ("deberta-base", "deberta", "encoding", "fp16", 0.999), + ("distilbert-base-uncased", "distilbert", "encoding", "fp16", 0.999), + ("dpr-ctx-encoder", "dpr", "encoding", "fp16", 0.99), + ("modernbert-base", "modernbert", "encoding", "fp32", 0.995), + ("roberta-base", "roberta", "encoding", "fp16", 0.999), + ], +) +def test_restored_encoder_profiles_are_family_local( + model: str, + family: str, + task: str, + precision: str, + minimum_cosine: float, +) -> None: + cases = select(discover(REPOSITORY), [model]) + + assert {case.kind for case in cases} == {"accuracy", "performance"} + assert all(case.family == family for case in cases) + assert all(case.source.parent == REPOSITORY / "families" / family / "tests/benchmark" for case in cases) + accuracy = next(case for case in cases if case.kind == "accuracy") + performance = next(case for case in cases if case.kind == "performance") + assert accuracy.candidate["task"] == task + assert accuracy.candidate["precision"] == precision + assert accuracy.values["gate"]["min_vector_cosine"] == minimum_cosine + assert accuracy.values["gate"]["min_vector_pass_rate"] == 1.0 + assert accuracy.values["gate"]["max_pair_cosine_abs_delta"] == 0.02 + if model == "dpr-ctx-encoder": + assert accuracy.values["reference"]["model_class"] == "dpr-context-encoder" + assert performance.benchmark == ( + "embedding_performance" if task == "embedding" else "encoder_performance" + ) + assert performance.values["operation"] == ("embed" if task == "embedding" else "encode") + if task == "embedding": + assert performance.values["reference"] == { + "runner": "task-reference", + "adapter": "hf-transformers-embedding", + "mode": "hf-eager", + "reference_backend": "hf_transformers", + "precision": "fp32", + "output_contract": "embedding-shape", + } + + def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> None: cases = select(discover(REPOSITORY), ["opt-125m"]) accuracy = next(case for case in cases if case.kind == "accuracy") @@ -354,7 +426,7 @@ def convert_ids_to_tokens(value: int) -> str: return {256047: "eng_Latn", 256057: "fra_Latn"}[value] tokenizer = Tokenizer() - controls = hf_text_generation._translation_controls( + controls, source_token_id = hf_text_generation._translation_controls( tokenizer, { "source_language": "eng_Latn", @@ -366,6 +438,44 @@ def convert_ids_to_tokens(value: int) -> str: assert tokenizer.src_lang == "eng_Latn" assert controls == {"forced_bos_token_id": 256057} + assert source_token_id is None + + +@pytest.mark.parametrize("runner", [hf_text_generation, hf_transformers]) +def test_hf_translation_supports_transformers5_generic_nllb_tokenizer(runner) -> None: + import torch + + class GenericNllbTokenizer: + unk_token_id = 3 + + @staticmethod + def convert_tokens_to_ids(value: str) -> int: + return {"eng_Latn": 256047, "fra_Latn": 256057}.get(value, 3) + + @staticmethod + def convert_ids_to_tokens(value: int) -> str: + return {256047: "eng_Latn", 256057: "fra_Latn"}[value] + + tokenizer = GenericNllbTokenizer() + controls, source_token_id = runner._translation_controls( + tokenizer, + { + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + }, + ) + encoded = { + "input_ids": torch.tensor([[17, 2, 3]]), + "attention_mask": torch.tensor([[1, 1, 1]]), + } + + runner._apply_source_language(encoded, source_token_id, tokenizer) + + assert controls == {"forced_bos_token_id": 256057} + assert source_token_id == 256047 + assert encoded["input_ids"].tolist() == [[17, 2, 256047]] def test_hf_accuracy_reference_normalizes_seq2seq_control_tokens() -> None: @@ -383,6 +493,204 @@ def test_hf_accuracy_reference_normalizes_seq2seq_control_tokens() -> None: ) == [17, 1] +def test_sts_samples_expand_pairs_with_family_owned_prompt_prefix(tmp_path: Path) -> None: + dataset = tmp_path / "sts.jsonl" + dataset.write_text( + "\n".join( + ( + json.dumps( + { + "genre": "captions", + "score": 2.5, + "sentence1": "A girl is styling her hair.", + "sentence2": "A girl is brushing her hair.", + } + ), + json.dumps( + { + "genre": "news", + "score": 4.0, + "sentence1": "One sentence.", + "sentence2": "Another sentence.", + } + ), + ) + ) + + "\n", + encoding="utf-8", + ) + + assert qualification_accuracy._sts_samples(dataset, 1, "query: ") == [ + { + "sample_id": "stsbenchmark-000000-a", + "pair_id": "stsbenchmark-000000", + "pair_side": "sentence1", + "score": 2.5, + "prompt": "query: A girl is styling her hair.", + }, + { + "sample_id": "stsbenchmark-000000-b", + "pair_id": "stsbenchmark-000000", + "pair_side": "sentence2", + "score": 2.5, + "prompt": "query: A girl is brushing her hair.", + }, + ] + + +def test_encoder_embedding_comparison_restores_pre_refactor_gates() -> None: + reference = [ + { + "sample_id": "pair-a", + "pair_id": "pair", + "pair_side": "sentence1", + "score": 5.0, + "vector": [1.0, 0.0], + }, + { + "sample_id": "pair-b", + "pair_id": "pair", + "pair_side": "sentence2", + "score": 5.0, + "vector": [0.8, 0.6], + }, + ] + candidate = [{"values": [1.0, 0.0]}, {"values": [0.8, 0.6]}] + + result = qualification_accuracy._compare_encoder_embeddings( + reference, + candidate, + { + "min_vector_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "max_pair_cosine_abs_delta": 0.02, + }, + ) + + assert result["status"] == "passed" + assert result["metrics"]["vector_pass_rate"] == 1.0 + assert result["metrics"]["max_pair_cosine_abs_delta"] == pytest.approx(0.0) + assert result["metrics"]["hf_sts_spearman"] is None + assert result["metrics"]["candidate_sts_spearman"] is None + + +@pytest.mark.parametrize( + ("task", "expected_mode", "expected_operation"), + [("encoding", "cls", "encode"), ("embedding", "embedding", "embed")], +) +def test_encoder_accuracy_uses_task_semantics_without_model_specific_runner( + tmp_path: Path, + monkeypatch, + task: str, + expected_mode: str, + expected_operation: str, +) -> None: + dataset_path = tmp_path / "sts.jsonl" + dataset_path.write_text( + json.dumps( + { + "genre": "captions", + "score": 3.0, + "sentence1": "Sentence one.", + "sentence2": "Sentence two.", + } + ) + + "\n", + encoding="utf-8", + ) + case = QualificationCase( + kind="accuracy", + model="encoder-model", + family="encoder", + name="stsbenchmark-parity", + benchmark="stsbenchmark_embedding_parity", + candidate={ + "family": "encoder", + "checkpoint": "org/model", + "task": task, + "precision": "fp16", + "build": {"max_sequence_length": 128}, + }, + values={ + "samples": 1, + "reference": {"precision": "fp32"}, + "gate": { + "min_vector_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "max_pair_cosine_abs_delta": 0.02, + }, + }, + source=tmp_path / "encoder.yaml", + reference_requirements=None, + ) + dataset = Dataset("stsbenchmark-test", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured_reference: dict[str, object] = {} + captured_candidate: dict[str, object] = {} + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured_reference.update(json.loads(request.read_text(encoding="utf-8"))) + samples = captured_reference["samples"] + output.write_text( + json.dumps( + { + "samples": [ + {**sample, "vector": [1.0, float(index)]} + for index, sample in enumerate(samples) + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def candidate(_case, _context, _output, operation, requests): + captured_candidate.update({"operation": operation, "requests": requests}) + return ( + [{"values": [1.0, float(index)]} for index, _request in enumerate(requests)], + tmp_path / "encoder.bundle", + ) + + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr(qualification_accuracy, "_candidate_outputs", candidate) + + result = qualification_accuracy._encoder_embedding_parity( + case, context, dataset, tmp_path / "output" + ) + + assert result["status"] == "passed" + assert captured_reference["mode"] == expected_mode + assert captured_candidate["operation"] == expected_operation + assert [request["request"] for request in captured_candidate["requests"]] == [ + {"prompt": "Sentence one."}, + {"prompt": "Sentence two."}, + ] + + +def test_hf_encoder_rejects_unknown_vector_mode() -> None: + with pytest.raises(ValueError, match="unsupported encoder vector mode"): + hf_encoder._vector_mode("mean") + + def test_shared_definitions_own_dataset_and_metric_not_models() -> None: for case in discover(REPOSITORY): definition = load_benchmark(REPOSITORY, case) From 7f206082b21a9a320c78acc3b2f6d9dd3984b06f Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 18:06:44 +0000 Subject: [PATCH 017/100] fix(benchmark): align embedding timing Compile embedding references before warmup and keep host materialization inside the timed operation. Summarize outputs after sampling so TensorRT and Hugging Face measure equivalent public-call boundaries while retaining the eager fallback. Signed-off-by: chaofengw --- .../performance/baselines/task_reference.py | 20 +++- .../trtmc_benchmark/tests/test_perf_matrix.py | 100 ++++++++++++++++-- .../tests/benchmark/all-minilm-l6-v2.yaml | 4 +- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index 849bd7045a..3d92fc6ab9 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -105,12 +105,13 @@ class Session: """One loaded reference model and its repeatable timed operation.""" - invoke: Callable[[], Mapping[str, Any]] + invoke: Callable[[], Any] framework: str timing_scope: str = "task-model-call-wall" input_preparation_included: bool = False asset_loading_included: bool = False compile_evidence: dict[str, Any] | None = None + summarize: Callable[[Any], Mapping[str, Any]] | None = None def build_parser() -> argparse.ArgumentParser: @@ -884,6 +885,9 @@ def _load_embedding( .eval() .to(device) ) + compile_evidence = ( + _compile_forward(model) if arguments.mode == "torch-compile" else None + ) prompt = prompts[0] def prepare_inputs() -> Mapping[str, Any]: @@ -896,7 +900,7 @@ def prepare_inputs() -> Mapping[str, Any]: if not declared_timing["input_preparation_included"]: prepared_inputs = prepare_inputs() - def invoke() -> Mapping[str, Any]: + def invoke() -> Any: inputs = prepare_inputs() if prepared_inputs is None else prepared_inputs with torch.inference_mode(): outputs = model(**inputs, output_hidden_states=True) @@ -907,6 +911,9 @@ def invoke() -> Mapping[str, Any]: mask = mask.unsqueeze(-1).to(hidden.dtype) vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) vector = torch.nn.functional.normalize(vector, p=2, dim=-1) + return vector.detach().to("cpu", dtype=torch.float32) + + def summarize(vector: Any) -> Mapping[str, Any]: summary = _tensor_summary(vector) summary.update( { @@ -923,6 +930,8 @@ def invoke() -> Mapping[str, Any]: timing_scope=str(declared_timing["timing_scope"]), input_preparation_included=bool(declared_timing["input_preparation_included"]), asset_loading_included=False, + compile_evidence=compile_evidence, + summarize=summarize, ) @@ -2342,7 +2351,7 @@ def compile_graph(graph: Any, inputs: Any, **options: Any) -> Any: def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: - output: Mapping[str, Any] = {} + output: Any = {} for _ in range(warmup): output = session.invoke() _synchronize() @@ -2363,7 +2372,8 @@ def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float and int(session.compile_evidence["compiled_graph_count"]) != compiled_graphs ): raise RuntimeError("model compilation occurred inside timed samples") - return samples, dict(output) + summary = session.summarize(output) if session.summarize is not None else output + return samples, dict(summary) def _run_elf( @@ -2781,6 +2791,8 @@ def run(arguments: argparse.Namespace) -> int: supported_modes = {expected_mode} if arguments.adapter == "pytorch-timeseries" and arguments.family == "chronos_bolt": supported_modes.add("torch-compile") + if arguments.adapter == "hf-transformers-embedding": + supported_modes.add("torch-compile") if arguments.mode not in supported_modes: raise ValueError( f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}" diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 1a71e33fbc..cbe474e412 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -1449,16 +1449,83 @@ def test_release_suite_expands_profiles_and_covers_ready_catalog() -> None: "family", "expected_scope", "input_preparation_included", + "mode", "calls_after_load", "calls_after_invoke", + "calls_after_summary", + "compiled", ), [ - ("bert", "task-pipeline-call-wall", True, [], ["tokenize", "model"]), - ("eagle_vlm", "task-model-call-wall", False, ["tokenize"], ["tokenize", "model"]), - ("bert", "task-model-call-wall", False, ["tokenize"], ["tokenize", "model"]), - ("eagle_vlm", "task-pipeline-call-wall", True, [], ["tokenize", "model"]), - ("renamed_embedding", "task-model-call-wall", False, ["tokenize"], ["tokenize", "model"]), - ("renamed_embedding", "task-pipeline-call-wall", True, [], ["tokenize", "model"]), + ( + "bert", + "task-pipeline-call-wall", + True, + "hf-eager", + [], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "bert", + "task-pipeline-call-wall", + True, + "torch-compile", + ["compile"], + ["compile", "tokenize", "model", "materialize"], + ["compile", "tokenize", "model", "materialize", "validate"], + True, + ), + ( + "eagle_vlm", + "task-model-call-wall", + False, + "hf-eager", + ["tokenize"], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "bert", + "task-model-call-wall", + False, + "hf-eager", + ["tokenize"], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "eagle_vlm", + "task-pipeline-call-wall", + True, + "hf-eager", + [], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "renamed_embedding", + "task-model-call-wall", + False, + "hf-eager", + ["tokenize"], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "renamed_embedding", + "task-pipeline-call-wall", + True, + "hf-eager", + [], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), ], ) def test_embedding_reference_measures_the_declared_timing_contract( @@ -1466,8 +1533,11 @@ def test_embedding_reference_measures_the_declared_timing_contract( family, expected_scope, input_preparation_included, + mode, calls_after_load, calls_after_invoke, + calls_after_summary, + compiled, ) -> None: calls: list[str] = [] @@ -1478,6 +1548,10 @@ class FakeTensor: def to(self, *_args, **_kwargs): return self + def detach(self): + calls.append("materialize") + return self + def unsqueeze(self, _dimension): return self @@ -1491,6 +1565,7 @@ def numel(self): return 2 def isfinite(self): + calls.append("validate") return self def all(self): @@ -1546,9 +1621,16 @@ def __call__(self, **_kwargs): fake_transformers.AutoTokenizer = FakeTokenizer monkeypatch.setitem(sys.modules, "torch", fake_torch) monkeypatch.setitem(sys.modules, "transformers", fake_transformers) + compile_evidence = {"compiled_graph_count": 0} + monkeypatch.setattr( + task_reference, + "_compile_forward", + lambda _model: calls.append("compile") or compile_evidence, + ) arguments = SimpleNamespace( family=family, model="sentence-transformers/all-MiniLM-L6-v2", + mode=mode, precision="fp32", revision="model-revision", trust_remote_code=False, @@ -1570,8 +1652,12 @@ def __call__(self, **_kwargs): assert session.timing_scope == expected_scope assert session.input_preparation_included is input_preparation_included assert session.asset_loading_included is False - assert session.invoke()["embedding_vectors"] == 1 + assert (session.compile_evidence is compile_evidence) is compiled + vector = session.invoke() assert calls == calls_after_invoke + assert session.summarize is not None + assert session.summarize(vector)["embedding_vectors"] == 1 + assert calls == calls_after_summary def test_check_resolves_selected_entry_with_one_runtime_root(tmp_path: Path, capsys) -> None: diff --git a/families/bert/tests/benchmark/all-minilm-l6-v2.yaml b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml index e2ea697916..a6f5ee84a5 100644 --- a/families/bert/tests/benchmark/all-minilm-l6-v2.yaml +++ b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml @@ -35,7 +35,9 @@ performance: reference: runner: task-reference adapter: hf-transformers-embedding - mode: hf-eager + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward reference_backend: hf_transformers precision: fp32 output_contract: embedding-shape From d0d8f5786e0d8c6a8f1dccec6759bd962fa983f7 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 18:07:07 +0000 Subject: [PATCH 018/100] refactor(benchmark): keep model details family-local Replace central model inventories and the DPR-specific runner switch with generic contract tests and family-declared Transformers classes. Require immutable revisions for trusted remote code, pin reviewed checkpoints, and ignore alternate framework weight variants generically for reproducible offline builds. Signed-off-by: chaofengw --- .../tensorrt_model_connect/model_support.py | 2 +- core/builder/tests/test_build_cli.py | 2 +- .../dpr/tests/benchmark/dpr-ctx-encoder.yaml | 3 +- families/glm/tests/benchmark/glm-4-9b.yaml | 1 + .../tests/benchmark/internlm2-1.8b.yaml | 1 + .../tests/benchmark/minitron-4b-depth.yaml | 1 + .../tests/benchmark/nemotron-hindi-4b.yaml | 1 + tools/benchmark_qualification/accuracy.py | 11 +- tools/benchmark_qualification/catalog.py | 7 + .../references/hf_encoder.py | 30 +- tools/tests/test_model_benchmark.py | 363 ++++++++++-------- 11 files changed, 239 insertions(+), 183 deletions(-) diff --git a/core/builder/tensorrt_model_connect/model_support.py b/core/builder/tensorrt_model_connect/model_support.py index 59e564f518..04fd18c307 100644 --- a/core/builder/tensorrt_model_connect/model_support.py +++ b/core/builder/tensorrt_model_connect/model_support.py @@ -165,7 +165,7 @@ def resolve_model(model: str, revision: str | None = None) -> Path: snapshot_download( repo_id=model, revision=revision, - ignore_patterns=["flax_model.msgpack", "tf_model.h5"], + ignore_patterns=["*flax_model*", "*tf_model*"], ) ) diff --git a/core/builder/tests/test_build_cli.py b/core/builder/tests/test_build_cli.py index be216c871f..8086a1524b 100644 --- a/core/builder/tests/test_build_cli.py +++ b/core/builder/tests/test_build_cli.py @@ -342,7 +342,7 @@ def snapshot_download(**kwargs): { "repo_id": "openai-community/gpt2", "revision": "revision-1", - "ignore_patterns": ["flax_model.msgpack", "tf_model.h5"], + "ignore_patterns": ["*flax_model*", "*tf_model*"], } ] diff --git a/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml index 0df5442ce2..1d5ec2a480 100644 --- a/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml +++ b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml @@ -18,7 +18,8 @@ accuracy: samples: 50 reference: precision: fp32 - model_class: dpr-context-encoder + model_class: transformers.DPRContextEncoder + tokenizer_class: transformers.DPRContextEncoderTokenizerFast gate: min_vector_cosine: 0.99 min_vector_pass_rate: 1.0 diff --git a/families/glm/tests/benchmark/glm-4-9b.yaml b/families/glm/tests/benchmark/glm-4-9b.yaml index 33e0ce81d2..2593f65670 100644 --- a/families/glm/tests/benchmark/glm-4-9b.yaml +++ b/families/glm/tests/benchmark/glm-4-9b.yaml @@ -6,6 +6,7 @@ model: glm-4-9b candidate: family: glm checkpoint: THUDM/glm-4-9b-hf + revision: b44e98fcc8df0faba03a48b405356af6b91821e7 task: text_generation precision: fp16 bundle: glm-4-9b.bundle diff --git a/families/internlm/tests/benchmark/internlm2-1.8b.yaml b/families/internlm/tests/benchmark/internlm2-1.8b.yaml index d8feea4ebc..97845c085b 100644 --- a/families/internlm/tests/benchmark/internlm2-1.8b.yaml +++ b/families/internlm/tests/benchmark/internlm2-1.8b.yaml @@ -6,6 +6,7 @@ model: internlm2-1.8b candidate: family: internlm checkpoint: internlm/internlm2-math-plus-1_8b + revision: 998dd540b92711b855982c0318a39064a1d3ddc1 task: text_generation precision: fp16 trust_remote_code: true diff --git a/families/llama/tests/benchmark/minitron-4b-depth.yaml b/families/llama/tests/benchmark/minitron-4b-depth.yaml index 303cf45165..cbb5ee26a0 100644 --- a/families/llama/tests/benchmark/minitron-4b-depth.yaml +++ b/families/llama/tests/benchmark/minitron-4b-depth.yaml @@ -6,6 +6,7 @@ model: minitron-4b-depth candidate: family: llama checkpoint: nvidia/Llama-3.1-Minitron-4B-Depth-Base + revision: 47f17b2c73bbd07a4b643a0382970704d389f737 task: text_generation precision: fp16 bundle: minitron-4b-depth.bundle diff --git a/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml b/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml index 8c1d420ba7..1bf4939621 100644 --- a/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml +++ b/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml @@ -6,6 +6,7 @@ model: nemotron-hindi-4b candidate: family: nemotron checkpoint: nvidia/Nemotron-4-Mini-Hindi-4B-Base + revision: 05c6ef6d2ddaad06bce914990facaf08bc4c581e task: text_generation precision: bf16 bundle: nemotron-hindi-4b.bundle diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index cacc2b4878..9c1a1bb1f6 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -197,9 +197,13 @@ def _encoder_embedding_parity( raise QualificationError( f"embedding vector parity does not support candidate task {task!r}" ) from error - model_class = str(reference.get("model_class", "auto")) - if model_class not in {"auto", "dpr-context-encoder"}: - raise QualificationError(f"unsupported encoder reference model class {model_class!r}") + model_class = reference.get("model_class", "auto") + tokenizer_class = reference.get("tokenizer_class", "auto") + if not all( + isinstance(value, str) and value + for value in (model_class, tokenizer_class) + ): + raise QualificationError("encoder reference classes must be non-empty strings") reference_request = { "model": str(case.candidate["checkpoint"]), "revision": case.candidate.get("revision"), @@ -207,6 +211,7 @@ def _encoder_embedding_parity( "precision": str(reference.get("precision", "fp32")), "mode": mode, "model_class": model_class, + "tokenizer_class": tokenizer_class, "max_length": int(case.candidate.get("build", {}).get("max_sequence_length", 512)), "samples": samples, } diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index b28e3c98c5..0515dfc8da 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from pathlib import Path +import re from typing import Any, Mapping, Sequence import yaml @@ -129,6 +130,12 @@ def _candidate(raw: Any, family: str, path: Path) -> dict[str, Any]: trust_remote_code = value.get("trust_remote_code") if trust_remote_code is not None and not isinstance(trust_remote_code, bool): raise QualificationError(f"{path}: candidate.trust_remote_code must be a boolean") + if trust_remote_code and ( + not isinstance(revision, str) or re.fullmatch(r"[0-9a-fA-F]{40}", revision) is None + ): + raise QualificationError( + f"{path}: trusted remote code requires an immutable 40-character revision" + ) build = value.get("build", {}) if not isinstance(build, Mapping): raise QualificationError(f"{path}: candidate.build must be an object") diff --git a/tools/benchmark_qualification/references/hf_encoder.py b/tools/benchmark_qualification/references/hf_encoder.py index 44a56ded49..7f381437a3 100644 --- a/tools/benchmark_qualification/references/hf_encoder.py +++ b/tools/benchmark_qualification/references/hf_encoder.py @@ -31,7 +31,9 @@ def main() -> int: raise ValueError(f"unsupported reference precision {precision!r}") mode = _vector_mode(str(request.get("mode", "cls"))) model_class, tokenizer_class = _reference_classes( - transformers, str(request.get("model_class", "auto")) + transformers, + str(request.get("model_class", "auto")), + str(request.get("tokenizer_class", "auto")), ) load_options = _model_load_options(request) tokenizer_options = { @@ -108,15 +110,23 @@ def _vector_mode(value: str) -> str: return value -def _reference_classes(transformers_module: Any, value: str) -> tuple[Any, Any]: - if value == "auto": - return transformers_module.AutoModel, transformers_module.AutoTokenizer - if value == "dpr-context-encoder": - return ( - transformers_module.DPRContextEncoder, - transformers_module.DPRContextEncoderTokenizerFast, - ) - raise ValueError(f"unsupported encoder reference model class {value!r}") +def _reference_classes( + transformers_module: Any, model: str, tokenizer: str +) -> tuple[Any, Any]: + return ( + _transformers_class(transformers_module, model, "AutoModel"), + _transformers_class(transformers_module, tokenizer, "AutoTokenizer"), + ) + + +def _transformers_class(module: Any, value: str, automatic: str) -> Any: + name = automatic if value == "auto" else value.removeprefix("transformers.") + if value != "auto" and (not value.startswith("transformers.") or not name.isidentifier()): + raise ValueError(f"unsupported Transformers class {value!r}") + resolved = getattr(module, name, None) + if resolved is None: + raise ValueError(f"unsupported Transformers class {value!r}") + return resolved def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 1ee83947a3..34af864a77 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -9,6 +9,7 @@ from pathlib import Path import subprocess import sys +from types import SimpleNamespace import pytest @@ -17,9 +18,9 @@ from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification.catalog import ( QualificationCase, + QualificationError, discover, load_benchmark, - select, ) from tools.benchmark_qualification.datasets import Dataset, resolve_dataset from tools.benchmark_qualification.references import hf_encoder, hf_text_generation @@ -47,169 +48,119 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert not any("l0" in case.model.lower() for case in cases) -def test_one_model_file_owns_multiple_cases_without_testcase_indirection() -> None: - cases = select(discover(REPOSITORY), ["gpt2-125m"]) - - assert {case.kind for case in cases} == {"accuracy", "performance"} - assert {case.name for case in cases} == {"mmlu-continuation", "generate-64"} - for case in cases: - assert case.source.parent.name == "benchmark" - assert "testcase" not in case.values - assert "testcase" not in str(case.values) - assert case.candidate["checkpoint"] == "openai-community/gpt2" +def test_l0_configs_outside_the_benchmark_folder_are_not_discovered( + tmp_path: Path, +) -> None: + profile = tmp_path / "families/example/tests/l0/example.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + " task: text_generation\n" + " precision: fp16\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + assert discover(tmp_path) == () -def test_bert_benchmark_uses_shared_sts_accuracy_and_encoder_performance() -> None: - cases = select(discover(REPOSITORY), ["bert-base-uncased"]) - assert {case.kind for case in cases} == {"accuracy", "performance"} - accuracy = next(case for case in cases if case.kind == "accuracy") - performance = next(case for case in cases if case.kind == "performance") - assert accuracy.benchmark == "stsbenchmark_embedding_parity" - assert accuracy.values["samples"] == 50 - assert accuracy.values["gate"] == { - "min_vector_cosine": 0.999, - "min_vector_pass_rate": 1.0, - "max_pair_cosine_abs_delta": 0.02, - } - assert load_benchmark(REPOSITORY, accuracy)["metric"]["name"] == ( - "embedding_vector_parity" +@pytest.mark.parametrize("revision", [None, "main", "abc1234"]) +def test_trusted_remote_code_requires_an_immutable_revision( + tmp_path: Path, revision: str | None +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + revision_line = "" if revision is None else f" revision: {revision}\n" + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + f"{revision_line}" + " task: text_generation\n" + " precision: fp16\n" + " trust_remote_code: true\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", ) - assert performance.benchmark == "encoder_performance" - assert performance.values["operation"] == "encode" - assert performance.values["reference"]["mode"] == "torch-compile" - assert performance.values["reference"]["fallback"] == "hf-eager" - -@pytest.mark.parametrize( - ("model", "family", "task", "precision", "minimum_cosine"), - [ - ("albert-base", "albert", "encoding", "fp16", 0.99), - ("all-minilm-l6-v2", "bert", "embedding", "fp16", 0.999), - ("all-mpnet-base-v2", "mpnet", "encoding", "fp16", 0.999), - ("deberta-base", "deberta", "encoding", "fp16", 0.999), - ("distilbert-base-uncased", "distilbert", "encoding", "fp16", 0.999), - ("dpr-ctx-encoder", "dpr", "encoding", "fp16", 0.99), - ("modernbert-base", "modernbert", "encoding", "fp32", 0.995), - ("roberta-base", "roberta", "encoding", "fp16", 0.999), - ], -) -def test_restored_encoder_profiles_are_family_local( - model: str, - family: str, - task: str, - precision: str, - minimum_cosine: float, -) -> None: - cases = select(discover(REPOSITORY), [model]) + with pytest.raises(QualificationError, match="immutable 40-character revision"): + discover(tmp_path) - assert {case.kind for case in cases} == {"accuracy", "performance"} - assert all(case.family == family for case in cases) - assert all(case.source.parent == REPOSITORY / "families" / family / "tests/benchmark" for case in cases) - accuracy = next(case for case in cases if case.kind == "accuracy") - performance = next(case for case in cases if case.kind == "performance") - assert accuracy.candidate["task"] == task - assert accuracy.candidate["precision"] == precision - assert accuracy.values["gate"]["min_vector_cosine"] == minimum_cosine - assert accuracy.values["gate"]["min_vector_pass_rate"] == 1.0 - assert accuracy.values["gate"]["max_pair_cosine_abs_delta"] == 0.02 - if model == "dpr-ctx-encoder": - assert accuracy.values["reference"]["model_class"] == "dpr-context-encoder" - assert performance.benchmark == ( - "embedding_performance" if task == "embedding" else "encoder_performance" - ) - assert performance.values["operation"] == ("embed" if task == "embedding" else "encode") - if task == "embedding": - assert performance.values["reference"] == { - "runner": "task-reference", - "adapter": "hf-transformers-embedding", - "mode": "hf-eager", - "reference_backend": "hf_transformers", - "precision": "fp32", - "output_contract": "embedding-shape", - } +def test_one_model_file_owns_multiple_cases_without_testcase_indirection( + tmp_path: Path, +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-model.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example-model\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + " task: text_generation\n" + " precision: fp16\n" + "accuracy:\n" + " - name: continuation\n" + " benchmark: example_accuracy\n" + "performance:\n" + " - name: generate\n" + " benchmark: example_performance\n" + " operation: generate\n", + encoding="utf-8", + ) -def test_opt_uses_validated_profile_and_pre_refactor_performance_length() -> None: - cases = select(discover(REPOSITORY), ["opt-125m"]) - accuracy = next(case for case in cases if case.kind == "accuracy") - performance = next(case for case in cases if case.kind == "performance") - - assert accuracy.candidate["build"]["max_sequence_length"] == 256 - assert accuracy.values["prompt_token_limit"] == 192 - assert performance.name == "generate-10" - assert performance.values["request"]["max_new_tokens"] == 10 - - -def test_restored_text_profiles_preserve_pre_refactor_performance_lengths() -> None: - expected = { - "codegen-350m": 20, - "deepseek-v2-lite": 10, - "deepseek-v2-tiny": 10, - "distilgpt2": 12, - "falcon3-1b": 20, - "gemma-2-2b": 10, - "glm-4-9b": 20, - "granite-3.1-2b": 20, - "internlm2-1.8b": 20, - "marian-en-ru": 20, - "minitron-4b-depth": 20, - "minitron-4b-width": 20, - "mistral-7b": 10, - "nemotron-hindi-4b": 20, - "nllb-200-distilled-600m": 20, - "olmo2-1b": 8, - "phi3-mini": 10, - "qwen3-0.6b-fp16": 10, - "stablelm2-1.6b": 22, - "starcoder2-3b": 20, - "riva-translate-4b": 20, - "t5-small": 20, - } + cases = discover(tmp_path) - for model, tokens in expected.items(): - cases = select(discover(REPOSITORY), [model]) - performance = next(case for case in cases if case.kind == "performance") - assert performance.name == f"generate-{tokens}" - assert performance.values["request"]["max_new_tokens"] == tokens - - stablelm = select(discover(REPOSITORY), ["stablelm2-1.6b"])[0] - assert stablelm.candidate["build"]["fp32_layers"] == [23] - minitron_width = select(discover(REPOSITORY), ["minitron-4b-width"])[0] - assert minitron_width.candidate["build"] == { - "max_sequence_length": 131072, - "dynamic_kv_cache": True, - } - minitron_depth_cases = select(discover(REPOSITORY), ["minitron-4b-depth"]) - minitron_depth = next(case for case in minitron_depth_cases if case.kind == "accuracy") - assert minitron_depth.candidate["build"]["max_sequence_length"] == ( - minitron_depth.values["prompt_token_limit"] - + minitron_depth.values["request"]["max_new_tokens"] - + 1 - ) - internlm = select(discover(REPOSITORY), ["internlm2-1.8b"])[0] - assert internlm.reference_requirements == ( - REPOSITORY / "families/internlm/requirements.txt" - ).resolve() + assert {case.kind for case in cases} == {"accuracy", "performance"} + assert {case.name for case in cases} == {"continuation", "generate"} + for case in cases: + assert case.source.parent.name == "benchmark" + assert "testcase" not in case.values + assert "testcase" not in str(case.values) + assert case.candidate["checkpoint"] == "example/model" -@pytest.mark.parametrize( - ("model", "expected_options"), - [ - ("deepseek-v2-tiny", {"experts_implementation": "batched_mm"}), - ("internlm2-1.8b", {"trust_remote_code": True}), - ], -) -def test_mmlu_forwards_reference_model_load_options( +def test_accuracy_forwards_declared_reference_model_load_options( tmp_path: Path, monkeypatch, - model: str, - expected_options: dict[str, object], ) -> None: - case = next( - case - for case in select(discover(REPOSITORY), [model]) - if case.kind == "accuracy" + case = QualificationCase( + kind="accuracy", + model="example-model", + family="example", + name="continuation", + benchmark="mmlu_continuation", + candidate={ + "family": "example", + "checkpoint": "example/model", + "revision": "a" * 40, + "task": "text_generation", + "precision": "fp16", + "trust_remote_code": True, + "build": {"max_sequence_length": 16}, + }, + values={ + "samples": 1, + "prompt_token_limit": 8, + "truncation_side": "left", + "reference": { + "precision": "fp16", + "experts_implementation": "batched_mm", + }, + "request": {"max_new_tokens": 1, "temperature": 0.0}, + "gate": {"min_pass_rate": 1.0, "allowed_failures": 0}, + }, + source=tmp_path / "example.yaml", + reference_requirements=None, ) dataset_path = tmp_path / "mmlu.json" dataset_path.write_text( @@ -264,7 +215,9 @@ def reference(command, *_args, **_kwargs): result = qualification_accuracy.run_accuracy(case, context) assert result["status"] == "passed" - assert {name: captured[name] for name in expected_options} == expected_options + assert captured["revision"] == "a" * 40 + assert captured["trust_remote_code"] is True + assert captured["experts_implementation"] == "batched_mm" def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) -> None: @@ -691,6 +644,35 @@ def test_hf_encoder_rejects_unknown_vector_mode() -> None: hf_encoder._vector_mode("mean") +def test_hf_encoder_resolves_family_declared_transformers_classes() -> None: + auto_model = object() + auto_tokenizer = object() + custom_model = object() + custom_tokenizer = object() + transformers = SimpleNamespace( + AutoModel=auto_model, + AutoTokenizer=auto_tokenizer, + CustomEncoder=custom_model, + CustomTokenizer=custom_tokenizer, + ) + + assert hf_encoder._reference_classes(transformers, "auto", "auto") == ( + auto_model, + auto_tokenizer, + ) + assert hf_encoder._reference_classes( + transformers, + "transformers.CustomEncoder", + "transformers.CustomTokenizer", + ) == (custom_model, custom_tokenizer) + with pytest.raises(ValueError, match="unsupported Transformers class"): + hf_encoder._reference_classes( + transformers, + "dpr-context-encoder", + "transformers.CustomTokenizer", + ) + + def test_shared_definitions_own_dataset_and_metric_not_models() -> None: for case in discover(REPOSITORY): definition = load_benchmark(REPOSITORY, case) @@ -736,19 +718,35 @@ def test_internal_automation_is_separate_from_the_installed_benchmark() -> None: def test_candidate_descriptor_is_generated_from_public_build_inputs(tmp_path: Path) -> None: - case = select(discover(REPOSITORY), ["gpt2-125m"])[0] + case = QualificationCase( + kind="accuracy", + model="example-model", + family="example", + name="continuation", + benchmark="example_accuracy", + candidate={ + "family": "example", + "checkpoint": "example/model", + "revision": "a" * 40, + "task": "text_generation", + "precision": "fp16", + "trust_remote_code": True, + "build": {"max_sequence_length": 1024}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) descriptor = write_model_descriptor(case, tmp_path, {"prompt": "hello"}) - value = descriptor.read_text(encoding="utf-8") + value = json.loads(descriptor.read_text(encoding="utf-8")) - assert '"hf_id": "openai-community/gpt2"' in value - assert '"task": "text_generation"' in value - assert '"max_sequence_length": 1024' in value - assert "tests/manifests" not in value - - internlm = select(discover(REPOSITORY), ["internlm2-1.8b"])[0] - descriptor = write_model_descriptor(internlm, tmp_path, {"prompt": "hello"}) - assert json.loads(descriptor.read_text(encoding="utf-8"))["trust_remote_code"] is True + assert value["hf_id"] == "example/model" + assert value["hf_revision"] == "a" * 40 + assert value["task"] == "text_generation" + assert value["max_sequence_length"] == 1024 + assert value["trust_remote_code"] is True + assert "tests/manifests" not in descriptor.read_text(encoding="utf-8") def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> None: @@ -771,7 +769,25 @@ def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> def test_family_reference_environment_inherits_parent_venv_packages( tmp_path: Path, monkeypatch ) -> None: - case = select(discover(REPOSITORY), ["internlm2-1.8b"])[0] + requirements = tmp_path / "requirements.txt" + requirements.write_text("transformers==4.46.3\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="custom-environment-model", + family="example", + name="continuation", + benchmark="mmlu_continuation", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=requirements, + ) context = RuntimeContext( repository=REPOSITORY, artifacts=tmp_path / "artifacts", @@ -816,14 +832,27 @@ def complete(command, *_args, **_kwargs): def test_bundle_preparation_uses_the_selected_runtime( tmp_path: Path, monkeypatch ) -> None: - case = next( - case - for case in select(discover(REPOSITORY), ["gpt2-125m"]) - if case.kind == "accuracy" + case = QualificationCase( + kind="accuracy", + model="example-model", + family="example", + name="continuation", + benchmark="example_accuracy", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "bundle": "example.bundle", + "build": {"max_sequence_length": 16}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, ) runtime_root = tmp_path / "runtime" runtime_root.mkdir() - bundle = tmp_path / "gpt2.bundle" + bundle = tmp_path / "example.bundle" bundle.write_bytes(b"bundle") descriptor = tmp_path / "candidate-model.json" descriptor.write_text("{}\n", encoding="utf-8") From d6153b159531648c983e2677b01d974928675dff Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 18:20:14 +0000 Subject: [PATCH 019/100] fix(benchmark): prime compiled measurements Run one untimed invocation when a compiled reference requests zero warmups so graph compilation cannot enter the measured samples. Preserve zero-warmup behavior for eager references. Signed-off-by: chaofengw --- .../performance/baselines/task_reference.py | 3 +- .../trtmc_benchmark/tests/test_perf_matrix.py | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index 3d92fc6ab9..4caca326b6 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -2352,7 +2352,8 @@ def compile_graph(graph: Any, inputs: Any, **options: Any) -> Any: def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: output: Any = {} - for _ in range(warmup): + untimed_iterations = max(warmup, int(session.compile_evidence is not None)) + for _ in range(untimed_iterations): output = session.invoke() _synchronize() compiled_graphs = None diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index cbe474e412..19e239412a 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -1660,6 +1660,36 @@ def __call__(self, **_kwargs): assert calls == calls_after_summary +@pytest.mark.parametrize(("compiled", "expected_invocations"), [(False, 2), (True, 3)]) +def test_zero_warmup_keeps_compilation_outside_timed_samples( + monkeypatch, compiled, expected_invocations +) -> None: + invocations = 0 + compile_evidence = {"compiled_graph_count": 0} if compiled else None + + def invoke(): + nonlocal invocations + invocations += 1 + if compile_evidence is not None and invocations == 1: + compile_evidence["compiled_graph_count"] = 1 + return {"value": invocations} + + monkeypatch.setattr(task_reference, "_synchronize", lambda: None) + samples, output = task_reference._measure( + task_reference.Session( + invoke=invoke, + framework="test", + compile_evidence=compile_evidence, + ), + warmup=0, + iterations=2, + ) + + assert len(samples) == 2 + assert invocations == expected_invocations + assert output == {"value": expected_invocations} + + def test_check_resolves_selected_entry_with_one_runtime_root(tmp_path: Path, capsys) -> None: environment_path, _ = _environment(tmp_path) assert ( From 0b1311525e9449244115349b10513df8e14a9ae6 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 17 Sep 2026 19:02:23 +0000 Subject: [PATCH 020/100] test(runtime): preserve factory mismatch coverage Embedding and encoding are now valid fake-family tasks for benchmark worker coverage. Use a deliberately mismatched text-generation bundle to keep the loader's exact task contract test meaningful. Signed-off-by: chaofengw --- core/runtime/tests/fake_family.cpp | 4 ++++ core/runtime/tests/test_family_loader.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index f887e4fb3d..1e09a355e8 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -75,5 +75,9 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context return new FakeEmbedding(); if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) return new FakeForecast(context.backend, context.kv_cache_size_bytes); + if (context.reader.info().task == trtmc::ITextGeneration::kTask) { + // Deliberately violate the factory contract for the loader's mismatch test. + return new FakeForecast(context.backend, context.kv_cache_size_bytes); + } throw std::runtime_error("unsupported fake task"); } diff --git a/core/runtime/tests/test_family_loader.cpp b/core/runtime/tests/test_family_loader.cpp index f7bf734f50..73a6bba726 100644 --- a/core/runtime/tests/test_family_loader.cpp +++ b/core/runtime/tests/test_family_loader.cpp @@ -154,7 +154,7 @@ int main(int argc, char** argv) { check(load_throws(unsafe_bundle, runtime_root.string()), "unsafe family id rejected"); const auto mismatch_bundle = runtime_root / "mismatch.bundle"; - write_bundle(mismatch_bundle, "fake", "embedding"); + write_bundle(mismatch_bundle, "fake", "text_generation"); check(load_throws(mismatch_bundle, runtime_root.string()), "factory task must exactly match bundle task"); From cf8ef5ae8e14e0d9a9ff2978dc5bb3a5e70f88b4 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 03:16:19 +0000 Subject: [PATCH 021/100] fix(benchmark): align translation qualification Force Hugging Face Performance references to use greedy decoding, require the NLLB profile to declare its source-language token placement explicitly, and distinguish model contract mismatches from execution errors in qualification reports. Signed-off-by: chaofengw --- .../performance/baselines/hf_transformers.py | 15 ++ .../trtmc_benchmark/tests/test_perf_matrix.py | 29 ++++ .../benchmark/nllb-200-distilled-600m.yaml | 2 + tools/benchmark_qualification/accuracy.py | 6 +- tools/benchmark_qualification/performance.py | 55 ++++++- .../references/hf_text_generation.py | 14 ++ tools/perf_matrix.py | 15 +- tools/tests/test_model_benchmark.py | 155 +++++++++++++++++- 8 files changed, 280 insertions(+), 11 deletions(-) diff --git a/apps/benchmark/performance/baselines/hf_transformers.py b/apps/benchmark/performance/baselines/hf_transformers.py index 41fa8eb397..f81030f405 100644 --- a/apps/benchmark/performance/baselines/hf_transformers.py +++ b/apps/benchmark/performance/baselines/hf_transformers.py @@ -185,9 +185,14 @@ def _translation_controls( """Use tokenizer language APIs; fixed-pair models validate their declared pair.""" source = request.get("source_language") target = request.get("target_language") + source_placement = request.get("source_language_placement") for name, value in (("source_language", source), ("target_language", target)): if value is not None and (not isinstance(value, str) or not value): raise ValueError(f"request.{name} must be a nonempty language identifier") + if source_placement is not None and source_placement != "replace-final-unk": + raise ValueError( + "request.source_language_placement must be replace-final-unk when set" + ) def token_id(language: str) -> int: lookup = getattr(tokenizer, "get_lang_id", None) @@ -213,6 +218,10 @@ def explicit_id(name: str) -> int | None: source_id = explicit_id("source_language_token_id") target_id = explicit_id("forced_bos_token_id") + if source_placement is not None and source_id is None: + raise ValueError( + "request.source_language_placement requires source_language_token_id" + ) manual_source_id = None if source_id is not None: if source is not None and token_id(source) != source_id: @@ -225,6 +234,11 @@ def explicit_id(name: str) -> int | None: elif source == getattr(tokenizer, "source_lang", None): pass elif source_id is not None and token_id(source) == source_id: + if source_placement != "replace-final-unk": + raise ValueError( + "generic translation tokenizer requires explicit " + "source_language_placement=replace-final-unk" + ) manual_source_id = source_id else: raise ValueError("reference tokenizer does not support the requested source language") @@ -358,6 +372,7 @@ def encode_prompt() -> Any: generation: dict[str, Any] = { "do_sample": float(request.get("temperature", 0.0)) > 0.0, "max_new_tokens": max_new_tokens, + "num_beams": 1, "use_cache": True, "pad_token_id": tokenizer.pad_token_id, } diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 19e239412a..b323e3492d 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -200,6 +200,7 @@ def decode(self, tokens, **_kwargs): class Model: config = SimpleNamespace(decoder_start_token_id=0, eos_token_id=3) + generation_config = SimpleNamespace(num_beams=4) def generate(self, **kwargs): captured.update(kwargs) @@ -228,10 +229,38 @@ def generate(self, **kwargs): assert captured["input_ids"].tolist() == [[10, 17]] assert captured["forced_bos_token_id"] == 11 assert captured["max_new_tokens"] == 5 and captured["do_sample"] is False + assert captured["num_beams"] == 1 assert captured["repetition_penalty"] == 1.1 assert output["token_ids"] == [8] +def test_reference_only_source_language_placement_reaches_hf_runner( + tmp_path: Path, +) -> None: + _, environment = _environment(tmp_path) + _, entries, _ = perf.load_suite(SUITE) + original = next(row for row in entries if row["id"] == "m2m_100.generate") + spec = { + **original, + "baseline": { + **original["baseline"], + "source_language_placement": "replace-final-unk", + }, + } + entry = perf.resolve_entries([spec], environment)[0] + + command = perf.baseline_command(entry, environment, tmp_path / "reference.json") + request = json.loads(command[command.index("--request-json") + 1]) + + assert request["source_language_placement"] == "replace-final-unk" + assert not any( + "source_language_placement" in argument + for argument in perf.candidate_command( + entry, environment, tmp_path / "candidate", no_build=True + ) + ) + + @pytest.mark.parametrize( "entry_id,task,operation", [ diff --git a/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml index fdcae38c11..f3e0778d24 100644 --- a/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml +++ b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml @@ -22,6 +22,7 @@ accuracy: precision: fp16 task: seq2seq-lm output_token_policy: strip-start-and-eos + source_language_placement: replace-final-unk request: &greedy-translation max_new_tokens: 128 temperature: 0.0 @@ -59,5 +60,6 @@ performance: fallback: hf-eager compile_scope: model.forward output_token_policy: strip-start-and-eos + source_language_placement: replace-final-unk precision: fp16 output_contract: exact-token-ids diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 9c1a1bb1f6..269c67245e 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -78,6 +78,10 @@ def _text_generation_parity( raise QualificationError( f"unsupported Accuracy output token policy {output_token_policy!r}" ) + reference_generation = dict(candidate_request) + source_language_placement = reference.get("source_language_placement") + if source_language_placement is not None: + reference_generation["source_language_placement"] = source_language_placement reference_request = { "model": str(case.candidate["checkpoint"]), "revision": case.candidate.get("revision"), @@ -87,7 +91,7 @@ def _text_generation_parity( "output_token_policy": output_token_policy, "prompt_token_limit": int(configured.get("prompt_token_limit", 192)), "truncation_side": str(configured.get("truncation_side", "left")), - "generation": dict(candidate_request), + "generation": reference_generation, "samples": selected, } experts_implementation = reference.get("experts_implementation") diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py index a88f7ce732..4bfb96daa9 100644 --- a/tools/benchmark_qualification/performance.py +++ b/tools/benchmark_qualification/performance.py @@ -23,6 +23,37 @@ ) +_COMPLETED_COMPARISONS = frozenset({"green", "yellow", "red"}) + + +def _qualification_status( + process_returncode: int, + matrix: Mapping[str, Any], + row: Mapping[str, Any], +) -> tuple[str, str | None]: + row_error = row.get("error") + if isinstance(row_error, str) and row_error: + return "error", row_error + row_status = row.get("status") + matrix_status = matrix.get("status") + if row_status == "contract-mismatch": + return "failed", None + if ( + row_status in _COMPLETED_COMPARISONS + and matrix_status == "completed" + and process_returncode == 0 + ): + return "passed", None + comparison = row.get("comparison") + reason = comparison.get("reason") if isinstance(comparison, Mapping) else None + if not isinstance(reason, str) or not reason: + reason = ( + f"Performance matrix ended with row status {row_status!r}, " + f"matrix status {matrix_status!r}, and exit code {process_returncode}" + ) + return "error", reason + + def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[str, Any]: output = context.case_artifacts(case) output.mkdir(parents=True, exist_ok=True) @@ -149,16 +180,28 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st if not run_directories: raise QualificationError(f"Performance produced no result; see {output}") run_directory = run_directories[-1] - matrix = json.loads((run_directory / "results.json").read_text(encoding="utf-8")) + try: + matrix = json.loads((run_directory / "results.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise QualificationError( + f"Performance produced an unreadable result; see {run_directory}" + ) from error + if not isinstance(matrix, Mapping): + raise QualificationError(f"Performance produced an invalid result; see {run_directory}") rows = matrix.get("rows") - row = rows[0] if isinstance(rows, list) and len(rows) == 1 else {} - passed = completed.returncode == 0 and matrix.get("status") == "completed" - comparison = row.get("comparison", {}) if isinstance(row, Mapping) else {} + row = ( + rows[0] + if isinstance(rows, list) and len(rows) == 1 and isinstance(rows[0], Mapping) + else {} + ) + status, error = _qualification_status(completed.returncode, matrix, row) + comparison_value = row.get("comparison") + comparison = comparison_value if isinstance(comparison_value, Mapping) else {} result = { "schema_version": "trtmc.qualification-result/v1", "case": case.id, "kind": "performance", - "status": "passed" if passed else "failed", + "status": status, "model": case.model, "benchmark": case.benchmark, "matrix_run": str(run_directory), @@ -170,5 +213,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st }, "reference_attempts": row.get("reference_attempts", []) if isinstance(row, Mapping) else [], } + if error is not None: + result["error"] = error write_result(output, result) return result diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py index 92a92b7cff..c8dd869949 100644 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ b/tools/benchmark_qualification/references/hf_text_generation.py @@ -144,9 +144,14 @@ def _translation_controls( ) -> tuple[dict[str, int], int | None]: source = request.get("source_language") target = request.get("target_language") + source_placement = request.get("source_language_placement") for name, value in (("source_language", source), ("target_language", target)): if value is not None and (not isinstance(value, str) or not value): raise ValueError(f"generation.{name} must be a nonempty language identifier") + if source_placement is not None and source_placement != "replace-final-unk": + raise ValueError( + "generation.source_language_placement must be replace-final-unk when set" + ) def token_id(language: str) -> int: lookup = getattr(tokenizer, "get_lang_id", None) @@ -172,6 +177,10 @@ def explicit_id(name: str) -> int | None: source_id = explicit_id("source_language_token_id") target_id = explicit_id("forced_bos_token_id") + if source_placement is not None and source_id is None: + raise ValueError( + "generation.source_language_placement requires source_language_token_id" + ) manual_source_id = None if source_id is not None: if source is not None and token_id(source) != source_id: @@ -184,6 +193,11 @@ def explicit_id(name: str) -> int | None: elif source == getattr(tokenizer, "source_lang", None): pass elif source_id is not None and token_id(source) == source_id: + if source_placement != "replace-final-unk": + raise ValueError( + "generic translation tokenizer requires explicit " + "source_language_placement=replace-final-unk" + ) manual_source_id = source_id else: raise ValueError("reference tokenizer does not support the requested source language") diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 460f85aa2f..19fb62f4e6 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -374,6 +374,15 @@ def _validate_entry(entry: Mapping[str, Any]) -> None: fallback = baseline.get("fallback") if fallback is not None and (not isinstance(fallback, str) or not fallback): raise PerfMatrixError(f"entry {entry['id']} baseline.fallback must be a mode") + source_placement = baseline.get("source_language_placement") + if source_placement is not None and ( + baseline["runner"] != "hf-transformers" + or source_placement != "replace-final-unk" + ): + raise PerfMatrixError( + f"entry {entry['id']} baseline.source_language_placement must be " + "replace-final-unk on an hf-transformers reference" + ) if not isinstance(measurement, Mapping): raise PerfMatrixError(f"entry {entry['id']} requires measurement") warmup = measurement.get("warmup") @@ -840,7 +849,11 @@ def _baseline_command( baseline = entry.spec["baseline"] runner = str(baseline["runner"]) selected_mode = str(mode or _baseline_mode(baseline)) - request = json.dumps(flatten_config(entry.case.request), ensure_ascii=True, separators=(",", ":")) + request_value = flatten_config(entry.case.request) + source_placement = baseline.get("source_language_placement") + if source_placement is not None: + request_value["source_language_placement"] = source_placement + request = json.dumps(request_value, ensure_ascii=True, separators=(",", ":")) common = [ "--model", str(_adapter_options(entry, environment).get("model_id", entry.model.hf_id)), diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 34af864a77..fae6860f5b 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -16,6 +16,7 @@ from apps.benchmark.performance.baselines import hf_transformers from apps.benchmark.performance.baselines.timing_contracts import timing_contract from tools.benchmark_qualification import accuracy as qualification_accuracy +from tools.benchmark_qualification import performance as qualification_performance from tools.benchmark_qualification.catalog import ( QualificationCase, QualificationError, @@ -276,6 +277,7 @@ def test_accuracy_forwards_seq2seq_reference_contract_and_nested_dataset_input( "precision": "fp16", "task": "seq2seq-lm", "output_token_policy": "strip-start-and-eos", + "source_language_placement": "replace-final-unk", }, "request": { "max_new_tokens": 128, @@ -349,6 +351,9 @@ def candidate(_case, _context, _output, _operation, requests): assert result["status"] == "passed" assert captured_reference["task"] == "seq2seq-lm" assert captured_reference["output_token_policy"] == "strip-start-and-eos" + assert captured_reference["generation"]["source_language_placement"] == ( + "replace-final-unk" + ) assert captured_reference["samples"] == [ {"sample_id": "translation-0", "prompt": "The house is wonderful."} ] @@ -410,13 +415,20 @@ def convert_ids_to_tokens(value: int) -> str: return {256047: "eng_Latn", 256057: "fra_Latn"}[value] tokenizer = GenericNllbTokenizer() + request = { + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + } + with pytest.raises(ValueError, match="source_language_placement"): + runner._translation_controls(tokenizer, request) + controls, source_token_id = runner._translation_controls( tokenizer, { - "source_language": "eng_Latn", - "source_language_token_id": 256047, - "target_language": "fra_Latn", - "forced_bos_token_id": 256057, + **request, + "source_language_placement": "replace-final-unk", }, ) encoded = { @@ -431,6 +443,141 @@ def convert_ids_to_tokens(value: int) -> str: assert encoded["input_ids"].tolist() == [[17, 2, 256047]] +def test_nllb_profile_owns_its_reference_source_language_placement() -> None: + cases = [case for case in discover(REPOSITORY) if case.model == "nllb-200-distilled-600m"] + + assert {case.kind for case in cases} == {"accuracy", "performance"} + assert all( + case.values["reference"]["source_language_placement"] == "replace-final-unk" + for case in cases + ) + + +@pytest.mark.parametrize( + ("row", "expected_status", "expected_error"), + [ + ( + { + "status": "contract-mismatch", + "comparison": {"reason": "generated token ids differ"}, + "reference_attempts": [], + }, + "failed", + None, + ), + ( + {"status": "white", "error": "candidate command failed"}, + "error", + "candidate command failed", + ), + ( + { + "status": "white", + "error": "reference command and configured fallback failed", + "reference_attempts": [ + { + "mode": "torch-compile", + "exit_code": 1, + "fallback_reason": "reference command failed", + } + ], + }, + "error", + "reference command and configured fallback failed", + ), + ], +) +def test_performance_distinguishes_model_contract_failures_from_execution_errors( + tmp_path: Path, + monkeypatch, + row: dict[str, object], + expected_status: str, + expected_error: str | None, +) -> None: + case = QualificationCase( + kind="performance", + model="example-model", + family="example", + name="generate", + benchmark="text_generation_performance", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={ + "operation": "generate", + "request": {"prompt": "Hello", "max_new_tokens": 1}, + "measurement": {"warmup": 1, "iterations": 10}, + "reference": { + "runner": "hf-transformers", + "mode": "hf-eager", + "precision": "fp32", + "output_contract": "exact-token-ids", + }, + }, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=tmp_path / "runtime", + trtmc_bench=tmp_path / "trtmc-bench", + worker=tmp_path / "worker", + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + definition = { + "reference_timing": { + "timing_scope": "public_operation_call_wall", + "input_preparation_included": True, + "asset_loading_included": False, + }, + "stability": { + "samples": 10, + "max_half_median_change_percent": 5.0, + "median_band_percent": 5.0, + "minimum_samples_within_band": 8, + "retries": 1, + }, + } + + def run_matrix(command, *_args, **_kwargs): + run_directory = context.case_artifacts(case) / "matrix/run" + run_directory.mkdir(parents=True) + matrix_row = {"id": "qualification.example.generate", **row} + (run_directory / "results.json").write_text( + json.dumps({"status": "failed", "rows": [matrix_row]}), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + + monkeypatch.setattr(qualification_performance, "load_benchmark", lambda *_: definition) + monkeypatch.setattr( + qualification_performance, + "require_candidate", + lambda *_: (context.worker, context.runtime_root), + ) + monkeypatch.setattr( + qualification_performance, "reference_python", lambda *_: Path(sys.executable) + ) + monkeypatch.setattr(qualification_performance, "run_command", run_matrix) + + result = qualification_performance.run_performance(case, context) + + assert result["status"] == expected_status + assert result.get("error") == expected_error + + def test_hf_accuracy_reference_normalizes_seq2seq_control_tokens() -> None: assert hf_text_generation._normalize_seq2seq_tokens( [2, 256057, 1034, 248075, 2], From a03cf18230ca3e809b4c89565eb369cf983a3974 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 05:07:22 +0000 Subject: [PATCH 022/100] feat(benchmark): add more text qualification profiles Register twelve existing text models through family-local Accuracy and Performance profiles. Reuse the shared parity and timing contracts so the additional coverage does not add a central model list or model-specific runner behavior. Signed-off-by: chaofengw --- .../tests/benchmark/bge-small-en-v1.5.yaml | 41 ++++++++++++++ .../benchmark/multilingual-e5-small.yaml | 44 +++++++++++++++ ...paraphrase-multilingual-minilm-l12-v2.yaml | 41 ++++++++++++++ .../tests/benchmark/convbert-base.yaml | 41 ++++++++++++++ .../benchmark/electra-base-discriminator.yaml | 41 ++++++++++++++ families/fnet/tests/benchmark/fnet-base.yaml | 41 ++++++++++++++ .../tests/benchmark/nemotron-nano-4b.yaml | 55 +++++++++++++++++++ .../tests/benchmark/camembert-base.yaml | 41 ++++++++++++++ .../tests/benchmark/roberta-large.yaml | 41 ++++++++++++++ .../tests/benchmark/xlm-roberta-base.yaml | 41 ++++++++++++++ .../xlnet/tests/benchmark/xlnet-base.yaml | 41 ++++++++++++++ 11 files changed, 468 insertions(+) create mode 100644 families/bert/tests/benchmark/bge-small-en-v1.5.yaml create mode 100644 families/bert/tests/benchmark/multilingual-e5-small.yaml create mode 100644 families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml create mode 100644 families/convbert/tests/benchmark/convbert-base.yaml create mode 100644 families/electra/tests/benchmark/electra-base-discriminator.yaml create mode 100644 families/fnet/tests/benchmark/fnet-base.yaml create mode 100644 families/llama/tests/benchmark/nemotron-nano-4b.yaml create mode 100644 families/roberta/tests/benchmark/camembert-base.yaml create mode 100644 families/roberta/tests/benchmark/roberta-large.yaml create mode 100644 families/roberta/tests/benchmark/xlm-roberta-base.yaml create mode 100644 families/xlnet/tests/benchmark/xlnet-base.yaml diff --git a/families/bert/tests/benchmark/bge-small-en-v1.5.yaml b/families/bert/tests/benchmark/bge-small-en-v1.5.yaml new file mode 100644 index 0000000000..ba2ab50f22 --- /dev/null +++ b/families/bert/tests/benchmark/bge-small-en-v1.5.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: bge-small-en-v1.5 +candidate: + family: bert + checkpoint: BAAI/bge-small-en-v1.5 + task: encoding + precision: fp16 + bundle: bge-small-en-v1.5.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/multilingual-e5-small.yaml b/families/bert/tests/benchmark/multilingual-e5-small.yaml new file mode 100644 index 0000000000..649a3154ce --- /dev/null +++ b/families/bert/tests/benchmark/multilingual-e5-small.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: multilingual-e5-small +candidate: + family: bert + checkpoint: intfloat/multilingual-e5-small + revision: 614241f622f53c4eeff9890bdc4f31cfecc418b3 + task: embedding + precision: fp16 + bundle: multilingual-e5-small.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: embed + benchmark: embedding_performance + operation: embed + request: + prompt: "query: What is artificial intelligence?" + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-embedding + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + reference_backend: hf_transformers + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml b/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml new file mode 100644 index 0000000000..359ca090c1 --- /dev/null +++ b/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: paraphrase-multilingual-minilm-l12-v2 +candidate: + family: bert + checkpoint: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + task: encoding + precision: fp16 + bundle: paraphrase-multilingual-minilm-l12-v2.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/convbert/tests/benchmark/convbert-base.yaml b/families/convbert/tests/benchmark/convbert-base.yaml new file mode 100644 index 0000000000..a2ae194bee --- /dev/null +++ b/families/convbert/tests/benchmark/convbert-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: convbert-base +candidate: + family: convbert + checkpoint: YituTech/conv-bert-base + task: encoding + precision: fp16 + bundle: convbert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/electra/tests/benchmark/electra-base-discriminator.yaml b/families/electra/tests/benchmark/electra-base-discriminator.yaml new file mode 100644 index 0000000000..c711b0c926 --- /dev/null +++ b/families/electra/tests/benchmark/electra-base-discriminator.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: electra-base-discriminator +candidate: + family: electra + checkpoint: google/electra-base-discriminator + task: encoding + precision: fp16 + bundle: electra-base-discriminator.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/fnet/tests/benchmark/fnet-base.yaml b/families/fnet/tests/benchmark/fnet-base.yaml new file mode 100644 index 0000000000..33da47203e --- /dev/null +++ b/families/fnet/tests/benchmark/fnet-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: fnet-base +candidate: + family: fnet + checkpoint: google/fnet-base + task: encoding + precision: fp16 + bundle: fnet-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/llama/tests/benchmark/nemotron-nano-4b.yaml b/families/llama/tests/benchmark/nemotron-nano-4b.yaml new file mode 100644 index 0000000000..5c5e4cfd15 --- /dev/null +++ b/families/llama/tests/benchmark/nemotron-nano-4b.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-nano-4b +candidate: + family: llama + checkpoint: nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1 + task: text_generation + precision: fp16 + bundle: nemotron-nano-4b.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: true + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/roberta/tests/benchmark/camembert-base.yaml b/families/roberta/tests/benchmark/camembert-base.yaml new file mode 100644 index 0000000000..41c8fbabbf --- /dev/null +++ b/families/roberta/tests/benchmark/camembert-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: camembert-base +candidate: + family: roberta + checkpoint: almanach/camembert-base + task: encoding + precision: fp16 + bundle: camembert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: Bonjour le monde. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/roberta/tests/benchmark/roberta-large.yaml b/families/roberta/tests/benchmark/roberta-large.yaml new file mode 100644 index 0000000000..439cfffdf4 --- /dev/null +++ b/families/roberta/tests/benchmark/roberta-large.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: roberta-large +candidate: + family: roberta + checkpoint: FacebookAI/roberta-large + task: encoding + precision: fp16 + bundle: roberta-large.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/roberta/tests/benchmark/xlm-roberta-base.yaml b/families/roberta/tests/benchmark/xlm-roberta-base.yaml new file mode 100644 index 0000000000..92dd3dbc4d --- /dev/null +++ b/families/roberta/tests/benchmark/xlm-roberta-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xlm-roberta-base +candidate: + family: roberta + checkpoint: FacebookAI/xlm-roberta-base + task: encoding + precision: fp16 + bundle: xlm-roberta-base.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/xlnet/tests/benchmark/xlnet-base.yaml b/families/xlnet/tests/benchmark/xlnet-base.yaml new file mode 100644 index 0000000000..441f07473d --- /dev/null +++ b/families/xlnet/tests/benchmark/xlnet-base.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xlnet-base +candidate: + family: xlnet + checkpoint: xlnet/xlnet-base-cased + task: encoding + precision: fp16 + bundle: xlnet-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape From bd7a0305aa97c585b2529f0e2854bd74424c01a4 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 06:55:40 +0000 Subject: [PATCH 023/100] fix(benchmark): align Llama accuracy prompts Keep MMLU candidate prompts aligned with the plain-text Hugging Face reference and reserve one position for the tokenizer BOS token. Preserve chat templates for the performance workloads. Signed-off-by: chaofengw --- families/llama/tests/benchmark/nemotron-nano-4b.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/families/llama/tests/benchmark/nemotron-nano-4b.yaml b/families/llama/tests/benchmark/nemotron-nano-4b.yaml index 5c5e4cfd15..058997273e 100644 --- a/families/llama/tests/benchmark/nemotron-nano-4b.yaml +++ b/families/llama/tests/benchmark/nemotron-nano-4b.yaml @@ -10,7 +10,7 @@ candidate: precision: fp16 bundle: nemotron-nano-4b.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 accuracy: - name: mmlu-continuation @@ -28,7 +28,7 @@ accuracy: min_p: 0.0 seed: -1 repetition_penalty: 1.0 - use_chat_template: true + use_chat_template: false enable_thinking: false do_sample: false gate: @@ -43,6 +43,7 @@ performance: <<: *greedy-generation prompt: What is the capital of France? max_new_tokens: 20 + use_chat_template: true measurement: warmup: 5 iterations: 10 From b133c43c4e9cb190f98d5c450e8d5dfc98ca19ce Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 07:23:40 +0000 Subject: [PATCH 024/100] fix(llama): sanitize decoded UTF-8 Replace invalid or truncated byte-level decoder sequences with the Unicode replacement character so generated token IDs remain reportable through JSON. Signed-off-by: chaofengw --- families/llama/runtime/bpe_tokenizer.cpp | 51 ++++++++++++++++++- .../tests/cpp/test_llama_plugin_helpers.cpp | 22 ++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/families/llama/runtime/bpe_tokenizer.cpp b/families/llama/runtime/bpe_tokenizer.cpp index 53a7a3ab2a..860defc1df 100644 --- a/families/llama/runtime/bpe_tokenizer.cpp +++ b/families/llama/runtime/bpe_tokenizer.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,54 @@ inline std::string utf32_to_utf8(char32_t cp) { return r; } +std::string replace_invalid_utf8(const std::string& text) { + static constexpr std::string_view replacement = "\xef\xbf\xbd"; + std::string result; + result.reserve(text.size()); + std::size_t pos = 0; + while (pos < text.size()) { + const auto lead = static_cast(text[pos]); + if (lead < 0x80) { + result.push_back(text[pos++]); + continue; + } + + std::size_t length = 0; + if (lead >= 0xC2 && lead <= 0xDF) + length = 2; + else if (lead >= 0xE0 && lead <= 0xEF) + length = 3; + else if (lead >= 0xF0 && lead <= 0xF4) + length = 4; + else { + result.append(replacement); + ++pos; + continue; + } + + if (pos + length > text.size()) { + result.append(replacement); + break; + } + const auto second = static_cast(text[pos + 1]); + bool valid = (second & 0xC0) == 0x80; + if (length >= 3) + valid = valid && (static_cast(text[pos + 2]) & 0xC0) == 0x80; + if (length == 4) + valid = valid && (static_cast(text[pos + 3]) & 0xC0) == 0x80; + valid = valid && !(lead == 0xE0 && second < 0xA0) && !(lead == 0xED && second >= 0xA0) && + !(lead == 0xF0 && second < 0x90) && !(lead == 0xF4 && second >= 0x90); + if (!valid) { + result.append(replacement); + ++pos; + continue; + } + result.append(text, pos, length); + pos += length; + } + return result; +} + // Read one UTF-8 codepoint from raw bytes, advance ptr. Returns 0xFFFD on error. inline char32_t read_utf8(const char*& p, const char* end) { if (p >= end) @@ -1009,7 +1058,7 @@ class BpeTokenizer final : public ITokenizer { result.append(text, start, pos - start); } } - return result; + return replace_invalid_utf8(result); } // Generic pre-tokenizer used when no GPT-2 pattern is declared. diff --git a/families/llama/tests/cpp/test_llama_plugin_helpers.cpp b/families/llama/tests/cpp/test_llama_plugin_helpers.cpp index 8ed30ed094..f218fdfa3f 100644 --- a/families/llama/tests/cpp/test_llama_plugin_helpers.cpp +++ b/families/llama/tests/cpp/test_llama_plugin_helpers.cpp @@ -58,6 +58,18 @@ std::string tokenizer_json() { })"; } +std::string byte_level_tokenizer_json() { + return R"({ + "model": { + "type": "BPE", + "unk_token": "", + "vocab": {"": 0, "\u00e2": 1, "\u012b": 2, "\u00a5": 3}, + "merges": [] + }, + "decoder": {"type": "ByteLevel"} + })"; +} + std::filesystem::path write_tokenizer_bundle(std::string_view tokenizer) { char path[] = "/tmp/trtmc_llama_plugin_helpers_XXXXXX"; const int descriptor = mkstemp(path); @@ -103,11 +115,21 @@ void test_exact_special_frame_respects_add_special_false() { "exact post-processor frame respects add_special_tokens=false"); } +void test_byte_level_decode_replaces_incomplete_utf8() { + const std::string json = byte_level_tokenizer_json(); + const auto tokenizer = trtmc::CreateBpeTokenizer(json.data(), json.size(), false); + check(tokenizer->decode({1, 2}) == "\xef\xbf\xbd", + "byte-level decode replaces an incomplete UTF-8 suffix"); + check(tokenizer->decode({1, 2, 3}) == "\xe2\x89\xa5", + "byte-level decode preserves a complete UTF-8 sequence"); +} + } // namespace int main() { test_exact_special_frame_uses_bundle_tokenizer(); test_exact_special_frame_respects_add_special_false(); + test_byte_level_decode_replaces_incomplete_utf8(); if (failures != 0) std::cerr << failures << " Llama plugin-helper test(s) failed\n"; return failures; From dacf73693290c1f0ffe1de21bbf1826226484d81 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 08:14:58 +0000 Subject: [PATCH 025/100] feat(benchmark): qualify additional text models Restore family-owned Accuracy and Performance profiles for BART, Nemotron Mini, Phi-MoE, and two Qwen variants. Keep checkpoint, build, and reference behavior local to each model profile so discovery remains automatic. Signed-off-by: chaofengw --- families/bart/tests/benchmark/bart-base.yaml | 59 +++++++++++++++++++ .../tests/benchmark/nemotron-mini-4b.yaml | 56 ++++++++++++++++++ families/phi_moe/tests/benchmark/phi-moe.yaml | 58 ++++++++++++++++++ .../qwen/tests/benchmark/qwen3-0.6b-fp8.yaml | 56 ++++++++++++++++++ .../benchmark/qwen3-4b-instruct-2507.yaml | 56 ++++++++++++++++++ 5 files changed, 285 insertions(+) create mode 100644 families/bart/tests/benchmark/bart-base.yaml create mode 100644 families/nemotron/tests/benchmark/nemotron-mini-4b.yaml create mode 100644 families/phi_moe/tests/benchmark/phi-moe.yaml create mode 100644 families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml create mode 100644 families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml diff --git a/families/bart/tests/benchmark/bart-base.yaml b/families/bart/tests/benchmark/bart-base.yaml new file mode 100644 index 0000000000..ab5d2872a2 --- /dev/null +++ b/families/bart/tests/benchmark/bart-base.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: bart-base +candidate: + family: bart + checkpoint: facebook/bart-base + task: text_generation + precision: fp16 + bundle: bart-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: wmt14-en-de + benchmark: wmt14_en_de_translation + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp32 + task: seq2seq-lm + output_token_policy: strip-start-and-eos + request: &greedy-generation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start-and-eos + precision: fp32 + output_contract: exact-token-ids diff --git a/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml b/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml new file mode 100644 index 0000000000..52f4e0aaf5 --- /dev/null +++ b/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-mini-4b +candidate: + family: nemotron + checkpoint: nvidia/Nemotron-Mini-4B-Instruct + task: text_generation + precision: fp16 + bundle: nemotron-mini-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/phi_moe/tests/benchmark/phi-moe.yaml b/families/phi_moe/tests/benchmark/phi-moe.yaml new file mode 100644 index 0000000000..6a9d2a1076 --- /dev/null +++ b/families/phi_moe/tests/benchmark/phi-moe.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: phi-moe +candidate: + family: phi_moe + checkpoint: microsoft/Phi-tiny-MoE-instruct + task: text_generation + precision: fp16 + bundle: phi-moe.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: Name the largest ocean on Earth. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + experts_implementation: batched_mm + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml new file mode 100644 index 0000000000..fa220e695b --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-0.6b-fp8 +candidate: + family: qwen + checkpoint: Qwen/Qwen3-0.6B + task: text_generation + precision: fp16 + bundle: qwen3-0.6b-fp8-fp16base.bundle + build: + max_sequence_length: 257 + quantization: fp8 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-1 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: "What place is named in the title of the 1979 live album by rock legends Cheap Trick? Answer:" + max_new_tokens: 1 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml b/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml new file mode 100644 index 0000000000..461a13800e --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-4b-instruct-2507 +candidate: + family: qwen + checkpoint: Qwen/Qwen3-4B-Instruct-2507 + task: text_generation + precision: fp16 + bundle: qwen3-4b-instruct-2507.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of Japan? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids From c7c058de2ce71018f06c8399e7e1d23246a49ab5 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 08:26:46 +0000 Subject: [PATCH 026/100] feat(benchmark): qualify large text models Restore family-owned Accuracy and Performance profiles for GPT-OSS, Nemotron-H, Qwen MoE, Qwen3.5, and Qwen3.8. Pin trusted remote-code checkpoints and reuse Nemotron-H's family-specific reference requirements. Signed-off-by: chaofengw --- .../gpt_oss/tests/benchmark/gpt-oss-20b.yaml | 56 +++++++++++++++++ .../tests/benchmark/nemotron-h-nano-9b.yaml | 61 +++++++++++++++++++ .../qwen3_5/tests/benchmark/qwen35-9b.yaml | 58 ++++++++++++++++++ .../qwen3_8/tests/benchmark/qwen38-27b.yaml | 58 ++++++++++++++++++ .../tests/benchmark/qwen3-moe-30b-a3b.yaml | 59 ++++++++++++++++++ 5 files changed, 292 insertions(+) create mode 100644 families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml create mode 100644 families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml create mode 100644 families/qwen3_5/tests/benchmark/qwen35-9b.yaml create mode 100644 families/qwen3_8/tests/benchmark/qwen38-27b.yaml create mode 100644 families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml diff --git a/families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml b/families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml new file mode 100644 index 0000000000..395991d511 --- /dev/null +++ b/families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gpt-oss-20b +candidate: + family: gpt_oss + checkpoint: openai/gpt-oss-20b + revision: 6cee5e81ee83917806bbde320786a8fb61efebee + task: text_generation + precision: fp16 + bundle: gpt-oss-20b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: <|start|>system<|message|>You are a helpful assistant.<|end|><|start|>user<|message|>What is the capital of France?<|end|><|start|>assistant + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml b/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml new file mode 100644 index 0000000000..f21f84164c --- /dev/null +++ b/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-h-nano-9b +candidate: + family: nemotron_h + checkpoint: nvidia/NVIDIA-Nemotron-Nano-9B-v2 + revision: 6533e8de2c68e4536bf7c411d7a3ce5734111476 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: nemotron-h-nano-9b.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-9b.yaml b/families/qwen3_5/tests/benchmark/qwen35-9b.yaml new file mode 100644 index 0000000000..e6d3a8e00d --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-9b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-9b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-9B + revision: c202236235762e1c871ad0ccb60c8ee5ba337b9a + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-9b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_8/tests/benchmark/qwen38-27b.yaml b/families/qwen3_8/tests/benchmark/qwen38-27b.yaml new file mode 100644 index 0000000000..1c6aa382af --- /dev/null +++ b/families/qwen3_8/tests/benchmark/qwen38-27b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen38-27b +candidate: + family: qwen3_8 + checkpoint: Qwen/Qwen3.8-27B + revision: 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen38-27b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml b/families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml new file mode 100644 index 0000000000..2ac4e54460 --- /dev/null +++ b/families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-moe-30b-a3b +candidate: + family: qwen_moe + checkpoint: Qwen/Qwen3-30B-A3B + revision: ad44e777bcd18fa416d9da3bd8f70d33ebb85d39 + task: text_generation + precision: fp16 + bundle: qwen3-moe-30b-a3b.bundle + build: + max_sequence_length: 129 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 64 + truncation_side: left + reference: + precision: fp32 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the largest planet in our solar system? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + experts_implementation: batched_mm + precision: fp32 + output_contract: exact-token-ids From 441ab20c3673a267b99aca2bc1d9787cc30185ea Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 09:25:59 +0000 Subject: [PATCH 027/100] feat(benchmark): qualify current text models Add family-owned Accuracy and Performance profiles for four LFM2 variants, three Qwen3.5 variants, MiniCPM5, Qwen3.6, and K2-Horizon. Keep checkpoint revisions and K2's reference environment local to the owning families. Signed-off-by: chaofengw --- .../tests/benchmark/k2-horizon-7b.yaml | 60 +++++++++++++++++++ families/lfm2/tests/benchmark/lfm2-1.2b.yaml | 56 +++++++++++++++++ families/lfm2/tests/benchmark/lfm2-2.6b.yaml | 56 +++++++++++++++++ .../lfm2/tests/benchmark/lfm2-350m-fp16.yaml | 56 +++++++++++++++++ families/lfm2/tests/benchmark/lfm2-700m.yaml | 56 +++++++++++++++++ .../llama/tests/benchmark/minicpm5-2b.yaml | 57 ++++++++++++++++++ .../qwen3_5/tests/benchmark/qwen35-0.8b.yaml | 58 ++++++++++++++++++ .../qwen3_5/tests/benchmark/qwen35-2b.yaml | 58 ++++++++++++++++++ .../qwen3_5/tests/benchmark/qwen35-4b.yaml | 58 ++++++++++++++++++ .../qwen3_8/tests/benchmark/qwen36-27b.yaml | 58 ++++++++++++++++++ 10 files changed, 573 insertions(+) create mode 100644 families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml create mode 100644 families/lfm2/tests/benchmark/lfm2-1.2b.yaml create mode 100644 families/lfm2/tests/benchmark/lfm2-2.6b.yaml create mode 100644 families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml create mode 100644 families/lfm2/tests/benchmark/lfm2-700m.yaml create mode 100644 families/llama/tests/benchmark/minicpm5-2b.yaml create mode 100644 families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml create mode 100644 families/qwen3_5/tests/benchmark/qwen35-2b.yaml create mode 100644 families/qwen3_5/tests/benchmark/qwen35-4b.yaml create mode 100644 families/qwen3_8/tests/benchmark/qwen36-27b.yaml diff --git a/families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml b/families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml new file mode 100644 index 0000000000..a26cd9a284 --- /dev/null +++ b/families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: k2-horizon-7b +candidate: + family: k2_horizon + checkpoint: IFM/K2-Horizon-7B + revision: 586b03f0fd1fbbf2f13eeafc33749e95ae34dd10 + task: text_generation + precision: bf16 + trust_remote_code: true + bundle: k2-horizon-7b.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-4 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 4 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-1.2b.yaml b/families/lfm2/tests/benchmark/lfm2-1.2b.yaml new file mode 100644 index 0000000000..13811fc77a --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-1.2b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-1.2b +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-1.2B + revision: 40f3da0d0164913923aee9462c23077868b816a3 + task: text_generation + precision: fp16 + bundle: lfm2-1.2b-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-2.6b.yaml b/families/lfm2/tests/benchmark/lfm2-2.6b.yaml new file mode 100644 index 0000000000..cc0ce20ae6 --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-2.6b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-2.6b +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-2.6B + revision: 36ed799f7024ef169ba92ffe184821835e15cc66 + task: text_generation + precision: fp16 + bundle: lfm2-2.6b-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml b/families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml new file mode 100644 index 0000000000..301da1c28e --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-350m-fp16 +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-350M + revision: f37d3f5c8c5484bc01dad379a595cf4c68c4e70e + task: text_generation + precision: fp16 + bundle: lfm2-350m-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-12 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 12 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-700m.yaml b/families/lfm2/tests/benchmark/lfm2-700m.yaml new file mode 100644 index 0000000000..e3e797cc70 --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-700m.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-700m +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-700M + revision: 86f49fc9a3800c3a325b7320bde179c318062583 + task: text_generation + precision: fp16 + bundle: lfm2-700m-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/llama/tests/benchmark/minicpm5-2b.yaml b/families/llama/tests/benchmark/minicpm5-2b.yaml new file mode 100644 index 0000000000..b5cf7288bd --- /dev/null +++ b/families/llama/tests/benchmark/minicpm5-2b.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: minicpm5-2b +candidate: + family: llama + checkpoint: openbmb/MiniCPM5-2B + revision: 12a3808a956f869c767195e9266b59c4d21d92e2 + task: text_generation + precision: fp16 + bundle: minicpm5-2b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml b/families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml new file mode 100644 index 0000000000..081c2ace45 --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-0.8b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-0.8B + revision: 2fc06364715b967f1860aea9cf38778875588b17 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-0.8b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-2b.yaml b/families/qwen3_5/tests/benchmark/qwen35-2b.yaml new file mode 100644 index 0000000000..c73616e6ce --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-2b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-2b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-2B + revision: 15852e8c16360a2fea060d615a32b45270f8a8fc + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-2b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-4b.yaml b/families/qwen3_5/tests/benchmark/qwen35-4b.yaml new file mode 100644 index 0000000000..f9ffc656cc --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-4b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-4b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-4B + revision: 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_8/tests/benchmark/qwen36-27b.yaml b/families/qwen3_8/tests/benchmark/qwen36-27b.yaml new file mode 100644 index 0000000000..7c41261e14 --- /dev/null +++ b/families/qwen3_8/tests/benchmark/qwen36-27b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen36-27b +candidate: + family: qwen3_8 + checkpoint: Qwen/Qwen3.6-27B + revision: 6a9e13bd6fc8f0983b9b99948120bc37f49c13e9 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen36-27b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids From 2921b76ecb4c174e9cc77b010ed888af9646c827 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 10:07:27 +0000 Subject: [PATCH 028/100] feat(benchmark): qualify more text models Add family-owned Accuracy and Performance profiles for LFM2 BF16, Qwen3-MoE tiny-random, and S1 Mini. Keep discovery automatic and reuse the shared model-agnostic qualification contracts. Signed-off-by: chaofengw --- .../benchmark/lfm2-350m-bf16-model-card.yaml | 56 ++++++++++++++++++ .../benchmark/qwen3-moe-tiny-random.yaml | 59 +++++++++++++++++++ .../s1_mini/tests/benchmark/s1-mini-fp16.yaml | 59 +++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml create mode 100644 families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml create mode 100644 families/s1_mini/tests/benchmark/s1-mini-fp16.yaml diff --git a/families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml b/families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml new file mode 100644 index 0000000000..d7dd7abc68 --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-350m-bf16-model-card +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-350M + revision: f37d3f5c8c5484bc01dad379a595cf4c68c4e70e + task: text_generation + precision: bf16 + bundle: lfm2-350m-bf16-model-card.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-12 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 12 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml b/families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml new file mode 100644 index 0000000000..b3d6da5846 --- /dev/null +++ b/families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-moe-tiny-random +candidate: + family: qwen_moe + checkpoint: amd-quark/tiny-random-qwen3_moe + revision: ca2aaa9a82e9f86c09c2785b405932fe6c806c90 + task: text_generation + precision: fp16 + bundle: qwen3-moe-tiny-random.bundle + build: + max_sequence_length: 129 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 64 + truncation_side: left + reference: + precision: fp32 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the largest planet in our solar system? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + experts_implementation: batched_mm + precision: fp32 + output_contract: exact-token-ids diff --git a/families/s1_mini/tests/benchmark/s1-mini-fp16.yaml b/families/s1_mini/tests/benchmark/s1-mini-fp16.yaml new file mode 100644 index 0000000000..d838b7ab98 --- /dev/null +++ b/families/s1_mini/tests/benchmark/s1-mini-fp16.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: s1-mini-fp16 +candidate: + family: s1_mini + checkpoint: superwhisper/s1-mini + revision: 65f84bcda1d13df582c4a8443c1c5aa53c0c66db + task: text_generation + precision: fp16 + bundle: s1-mini-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: normalize-40 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: |- + [Styling: semi-formal] [Structure: prose] [Context: general] + so um i need to like send the the report by uh friday no wait make that thursday + max_new_tokens: 40 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids From 8863bf207c3c6171815191c0183e2c8ee8996e64 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 10:22:20 +0000 Subject: [PATCH 029/100] feat(benchmark): qualify time-series models Add family-owned ETTh1 Accuracy references and Performance profiles for PatchTSMixer, PatchTST, and TimesFM. Reuse the shared time-series mechanics while keeping checkpoint loading and reference semantics inside each family. Signed-off-by: chaofengw --- .../patchtsmixer-granite-official.yaml | 54 ++++++++++ .../patchtsmixer/tests/benchmark/reference.py | 89 +++++++++++++++++ ...atchtst-etth1-regression-distribution.yaml | 54 ++++++++++ .../benchmark/patchtst-granite-official.yaml | 52 ++++++++++ .../patchtst/tests/benchmark/reference.py | 99 +++++++++++++++++++ families/timesfm/tests/benchmark/reference.py | 95 ++++++++++++++++++ .../benchmark/timesfm-2.0-500m-official.yaml | 50 ++++++++++ 7 files changed, 493 insertions(+) create mode 100644 families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml create mode 100644 families/patchtsmixer/tests/benchmark/reference.py create mode 100644 families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml create mode 100644 families/patchtst/tests/benchmark/patchtst-granite-official.yaml create mode 100644 families/patchtst/tests/benchmark/reference.py create mode 100644 families/timesfm/tests/benchmark/reference.py create mode 100644 families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml diff --git a/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml b/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml new file mode 100644 index 0000000000..6542bd2612 --- /dev/null +++ b/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: patchtsmixer-granite-official +candidate: + family: patchtsmixer + checkpoint: ibm-granite/granite-timeseries-patchtsmixer + revision: 90dc5a88d45f032b7dceefb5d814ca2af54f2ff9 + task: time_series_forecast + precision: fp16 + bundle: patchtsmixer-granite-official.bundle + build: + fp32_layers: [0, 1, 2, 3, 12] + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [HUFL, HULL, MUFL, MULL, LUFL, LULL, OT] + context_length: 512 + prediction_length: 96 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 5.0e-4 + max_absolute_error: 2.5e-2 + +performance: + - name: forecast-96 + benchmark: time_series_performance + operation: solve + request: + past_values: + [0.1, -0.2, 0.05, 0.3, -0.15, 0.25, 0.4, 0.2, -0.1, 0.1, + 0.35, -0.05, 0.2, 0.5, 0.3, 0.0, 0.15, 0.4, 0.05, 0.15, + 0.6, 0.4, 0.1, 0.2, 0.45, 0.1, 0.1, 0.7] + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 5.0e-4 + max_absolute_error: 2.5e-2 diff --git a/families/patchtsmixer/tests/benchmark/reference.py b/families/patchtsmixer/tests/benchmark/reference.py new file mode 100644 index 0000000000..3626cb4478 --- /dev/null +++ b/families/patchtsmixer/tests/benchmark/reference.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""PatchTSMixer-owned ETTh1 Accuracy reference.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if request.get("precision") != "fp32": + raise ValueError("PatchTSMixer Accuracy requires an fp32 reference") + if not torch.cuda.is_available(): + raise RuntimeError("PatchTSMixer Accuracy reference requires CUDA") + options = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + config = transformers.AutoConfig.from_pretrained(str(request["model"]), **options) + model = ( + transformers.PatchTSMixerForPrediction.from_pretrained( + str(request["model"]), torch_dtype=torch.float32, **options + ) + .eval() + .to("cuda") + ) + context = int(config.context_length) + channels = int(config.num_input_channels) + samples = [] + with torch.inference_mode(): + for sample in request["samples"]: + raw = [float(value) for value in sample["past_values"]] + count = min(len(raw), context * channels) + values = [0.0] * (context * channels) + values[-count:] = raw[-count:] + observed = [0.0] * (context * channels) + observed[-count:] = [1.0] * count + output = model( + past_values=torch.tensor( + values, dtype=torch.float32, device="cuda" + ).reshape(1, context, channels), + observed_mask=torch.tensor( + observed, dtype=torch.float32, device="cuda" + ).reshape(1, context, channels), + return_loss=False, + return_dict=True, + ).prediction_outputs.detach().float().cpu() + samples.append( + { + "sample_id": str(sample["sample_id"]), + "shape": [int(size) for size in output.shape], + "values": output.reshape(-1).tolist(), + } + ) + arguments.output.write_text( + json.dumps( + { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": request.get("revision"), + "precision": "fp32", + "samples": samples, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml b/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml new file mode 100644 index 0000000000..18a49a6f96 --- /dev/null +++ b/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: patchtst-etth1-regression-distribution +candidate: + family: patchtst + checkpoint: ibm-research/patchtst-etth1-regression-distribution + revision: d5ef731ed80ffcd2f0a3262138b1009bbcedb2c4 + task: time_series_forecast + precision: fp16 + bundle: patchtst-etth1-regression-distribution.bundle + build: + fp32_layers: [5] + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [HUFL, HULL, MUFL, MULL, LUFL, LULL] + context_length: 512 + prediction_length: 1 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 1.0e-3 + max_absolute_error: 1.0e-3 + +performance: + - name: regress + benchmark: time_series_performance + operation: solve + request: + past_values: + [0.1, -0.2, 0.05, 0.3, -0.15, 0.25, 0.4, 0.2, -0.1, 0.1, + 0.35, -0.05, 0.2, 0.5, 0.3, 0.0, 0.15, 0.4, 0.05, 0.15, + 0.6, 0.4, 0.1, 0.2, 0.45, 0.1, 0.1, 0.7, 0.55, 0.05] + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 1.0e-3 + max_absolute_error: 1.0e-3 diff --git a/families/patchtst/tests/benchmark/patchtst-granite-official.yaml b/families/patchtst/tests/benchmark/patchtst-granite-official.yaml new file mode 100644 index 0000000000..977a29d238 --- /dev/null +++ b/families/patchtst/tests/benchmark/patchtst-granite-official.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: patchtst-granite-official +candidate: + family: patchtst + checkpoint: ibm-granite/granite-timeseries-patchtst + revision: 7fe295d8bc8fbac8041b60ab351882634165517f + task: time_series_forecast + precision: fp16 + bundle: patchtst-granite-official.bundle + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [HUFL, HULL, MUFL, MULL, LUFL, LULL, OT] + context_length: 512 + prediction_length: 96 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 1.5e-3 + max_absolute_error: 3.5e-2 + +performance: + - name: forecast-96 + benchmark: time_series_performance + operation: solve + request: + past_values: + [0.1, -0.2, 0.05, 0.3, -0.15, 0.25, 0.4, 0.2, -0.1, 0.1, + 0.35, -0.05, 0.2, 0.5, 0.3, 0.0, 0.15, 0.4, 0.05, 0.15, + 0.6, 0.4, 0.1, 0.2, 0.45, 0.1, 0.1, 0.7] + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 1.5e-3 + max_absolute_error: 3.5e-2 diff --git a/families/patchtst/tests/benchmark/reference.py b/families/patchtst/tests/benchmark/reference.py new file mode 100644 index 0000000000..ad2d36ff59 --- /dev/null +++ b/families/patchtst/tests/benchmark/reference.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""PatchTST-owned ETTh1 Accuracy reference.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if request.get("precision") != "fp32": + raise ValueError("PatchTST Accuracy requires an fp32 reference") + if not torch.cuda.is_available(): + raise RuntimeError("PatchTST Accuracy reference requires CUDA") + options = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + config = transformers.AutoConfig.from_pretrained(str(request["model"]), **options) + architecture = " ".join(getattr(config, "architectures", ()) or ()).lower() + if "regression" in architecture: + model_class = transformers.PatchTSTForRegression + output_name = "regression_outputs" + else: + model_class = transformers.PatchTSTForPrediction + output_name = "prediction_outputs" + model = ( + model_class.from_pretrained( + str(request["model"]), torch_dtype=torch.float32, **options + ) + .eval() + .to("cuda") + ) + context = int(config.context_length) + channels = int(config.num_input_channels) + samples = [] + with torch.inference_mode(): + for sample in request["samples"]: + raw = [float(value) for value in sample["past_values"]] + count = min(len(raw), context * channels) + values = [0.0] * (context * channels) + values[-count:] = raw[-count:] + observed = [False] * (context * channels) + observed[-count:] = [True] * count + outputs = model( + past_values=torch.tensor( + values, dtype=torch.float32, device="cuda" + ).reshape(1, context, channels), + past_observed_mask=torch.tensor( + observed, dtype=torch.bool, device="cuda" + ).reshape(1, context, channels), + return_dict=True, + ) + output = getattr(outputs, output_name) + if isinstance(output, tuple): + output = torch.stack(output, dim=-1) + output = output.detach().float().cpu() + samples.append( + { + "sample_id": str(sample["sample_id"]), + "shape": [int(size) for size in output.shape], + "values": output.reshape(-1).tolist(), + } + ) + arguments.output.write_text( + json.dumps( + { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": request.get("revision"), + "precision": "fp32", + "samples": samples, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/timesfm/tests/benchmark/reference.py b/families/timesfm/tests/benchmark/reference.py new file mode 100644 index 0000000000..4e9af1b9a9 --- /dev/null +++ b/families/timesfm/tests/benchmark/reference.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TimesFM-owned ETTh1 Accuracy reference.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if request.get("precision") != "fp32": + raise ValueError("TimesFM Accuracy requires an fp32 reference") + if not torch.cuda.is_available(): + raise RuntimeError("TimesFM Accuracy reference requires CUDA") + options = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + model = ( + transformers.TimesFmModelForPrediction.from_pretrained( + str(request["model"]), torch_dtype=torch.float32, **options + ) + .eval() + .to("cuda") + ) + context = int(model.config.context_length) + samples = [] + with torch.inference_mode(): + for sample in request["samples"]: + raw = [float(value) for value in sample["past_values"]] + count = min(len(raw), context) + values = [0.0] * context + values[-count:] = raw[-count:] + padding = [1] * context + padding[-count:] = [0] * count + decoded = model.decoder( + past_values=torch.tensor( + values, dtype=torch.float32, device="cuda" + ).reshape(1, context), + past_values_padding=torch.tensor( + padding, dtype=torch.int32, device="cuda" + ).reshape(1, context), + freq=torch.tensor( + [[int(sample.get("frequency", 0))]], + dtype=torch.long, + device="cuda", + ), + output_attentions=False, + output_hidden_states=False, + ) + output = model._postprocess_output( + decoded.last_hidden_state, (decoded.loc, decoded.scale) + )[:, -1, : model.config.horizon_length, 0].detach().float().cpu() + samples.append( + { + "sample_id": str(sample["sample_id"]), + "shape": [int(size) for size in output.shape], + "values": output.reshape(-1).tolist(), + } + ) + arguments.output.write_text( + json.dumps( + { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": request.get("revision"), + "precision": "fp32", + "samples": samples, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml b/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml new file mode 100644 index 0000000000..95176a8f27 --- /dev/null +++ b/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: timesfm-2.0-500m-official +candidate: + family: timesfm + checkpoint: google/timesfm-2.0-500m-pytorch + revision: dc2443792ce5516872b89b37cf1bc058c3bf0c10 + task: time_series_forecast + precision: fp32 + bundle: timesfm-2.0-500m-official.bundle + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [OT] + context_length: 2048 + prediction_length: 128 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 4.0e-3 + max_absolute_error: 7.0e-3 + +performance: + - name: forecast-128 + benchmark: time_series_performance + operation: solve + request: + past_values: [0.05, 0.15, 0.3, 0.5, 0.65, 0.8, 0.95, 1.1, 1.18, 1.24, 1.28, 1.31] + frequency: 2 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 4.0e-3 + max_absolute_error: 7.0e-3 From 5e987e9077c169fdac27d3f0368b26b3f05f635b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 17:51:07 +0000 Subject: [PATCH 030/100] feat(benchmark): qualify TIMM classifiers Restore user-facing Accuracy and Performance qualification for all non-L0 single-device TIMM classification manifests. Keep model registration family-local while sharing only the generic Imagenette parity and TIMM reference mechanics. Signed-off-by: chaofengw --- .../performance/baselines/task_reference.py | 12 +- .../convnext-tiny-in12k-ft-in1k.yaml | 42 +++++ .../tests/benchmark/crossvit-9-240-in1k.yaml | 42 +++++ .../tests/benchmark/densenet121-ra-in1k.yaml | 41 +++++ .../tests/benchmark/dpn68b-ra-in1k.yaml | 42 +++++ .../tests/benchmark/dpn92-mx-in1k.yaml | 42 +++++ .../benchmark/efficientnet-b0-ra-in1k.yaml | 41 +++++ .../tests/benchmark/ghostnet-100-in1k.yaml | 42 +++++ .../benchmark/hrnet-w18-ms-aug-in1k.yaml | 42 +++++ .../tests/benchmark/inception-v3-tv-in1k.yaml | 41 +++++ .../inception-resnet-v2-tf-in1k.yaml | 42 +++++ .../tests/benchmark/inception-v4-tf-in1k.yaml | 42 +++++ .../benchmark/mnasnet-100-rmsp-in1k.yaml | 41 +++++ .../benchmark/mobilenetv2-100-ra-in1k.yaml | 41 +++++ .../mobilenetv3-large-100-ra-in1k.yaml | 41 +++++ .../benchmark/mobilenetv4-conv-large.yaml | 41 +++++ .../benchmark/mobilenetv4-conv-medium.yaml | 41 +++++ .../benchmark/mobilenetv4-conv-small-050.yaml | 41 +++++ .../benchmark/mobilenetv4-conv-small.yaml | 41 +++++ .../benchmark/mobilevit-s-cvnets-in1k.yaml | 42 +++++ .../benchmark/mobilevit-xxs-cvnets-in1k.yaml | 42 +++++ .../tests/benchmark/dm-nfnet-f0-dm-in1k.yaml | 42 +++++ .../tests/benchmark/dm-nfnet-f3-dm-in1k.yaml | 42 +++++ .../tests/benchmark/regnety-040-ra3-in1k.yaml | 42 +++++ .../tests/benchmark/repvgg-a2-rvgg-in1k.yaml | 42 +++++ .../benchmark/res2net50-26w-4s-in1k.yaml | 42 +++++ .../tests/benchmark/res2net50d-in1k.yaml | 42 +++++ .../tests/benchmark/res2next50-in1k.yaml | 42 +++++ .../tests/benchmark/resnest50d-in1k.yaml | 42 +++++ .../tests/benchmark/resnet50-a1-in1k.yaml | 41 +++++ .../tests/benchmark/senet154-gluon-in1k.yaml | 42 +++++ .../tests/benchmark/seresnet50-a1-in1k.yaml | 42 +++++ .../swin-tiny-patch4-window7-224-ms-in1k.yaml | 42 +++++ .../tests/benchmark/vgg16-tv-in1k.yaml | 41 +++++ ...vit-base-p16-224-augreg-in21k-ft-in1k.yaml | 41 +++++ .../tests/benchmark/xception41-tf-in1k.yaml | 42 +++++ .../xcit-nano-12-p16-224-fb-in1k.yaml | 42 +++++ .../xcit-tiny-12-p16-224-fb-in1k.yaml | 42 +++++ tools/benchmark_qualification/accuracy.py | 157 ++++++++++++++++++ .../image_classification_performance.yaml | 15 ++ .../benchmarks/imagenette_classification.yaml | 20 +++ tools/benchmark_qualification/performance.py | 22 +++ .../references/timm_image_classification.py | 89 ++++++++++ tools/tests/test_architecture.py | 11 +- tools/tests/test_model_benchmark.py | 152 ++++++++++++++++- 45 files changed, 2011 insertions(+), 8 deletions(-) create mode 100644 families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml create mode 100644 families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml create mode 100644 families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml create mode 100644 families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml create mode 100644 families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml create mode 100644 families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml create mode 100644 families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml create mode 100644 families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml create mode 100644 families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml create mode 100644 families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml create mode 100644 families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml create mode 100644 families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml create mode 100644 families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml create mode 100644 families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml create mode 100644 families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml create mode 100644 families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml create mode 100644 families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml create mode 100644 families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml create mode 100644 families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml create mode 100644 families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml create mode 100644 families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml create mode 100644 families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml create mode 100644 families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml create mode 100644 families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml create mode 100644 families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml create mode 100644 families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml create mode 100644 families/timm_res2net/tests/benchmark/res2next50-in1k.yaml create mode 100644 families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml create mode 100644 families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml create mode 100644 families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml create mode 100644 families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml create mode 100644 families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml create mode 100644 families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml create mode 100644 families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml create mode 100644 families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml create mode 100644 families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml create mode 100644 families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml create mode 100644 tools/benchmark_qualification/benchmarks/image_classification_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_classification.yaml create mode 100644 tools/benchmark_qualification/references/timm_image_classification.py diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index 4caca326b6..d008a70825 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -80,6 +80,7 @@ "pytorch-lerobot-act", "pytorch-personaplex", "pytorch-timeseries", + "timm-classification", "upstream-elf", "upstream-fast-foundation-stereo", "upstream-lance", @@ -1522,7 +1523,7 @@ def _load_vision( kwargs = _load_kwargs(arguments, torch) processor_kwargs = _processor_kwargs(arguments) - if arguments.family in { + if arguments.adapter == "timm-classification" or arguments.family in { "timm_densenet", "timm_efficientnet", "timm_inception", @@ -1537,7 +1538,10 @@ def _load_vision( import timm from timm.data import create_transform, resolve_model_data_config - model = timm.create_model(f"hf-hub:{arguments.model}", pretrained=True) + model_id = arguments.model + if arguments.revision: + model_id += "@" + arguments.revision + model = timm.create_model(f"hf-hub:{model_id}", pretrained=True) dtype = _torch_dtype(torch, arguments.precision) model = model.eval().to(device=device, dtype=dtype) transform = create_transform(**resolve_model_data_config(model), is_training=False) @@ -1684,7 +1688,8 @@ def invoke() -> Mapping[str, Any]: "width": width, } - return Session(invoke, "transformers") + framework = "timm" if arguments.adapter == "timm-classification" else "transformers" + return Session(invoke, framework) def _qwen3_omni_chat_inputs(processor: Any, conversation: Sequence[Mapping[str, Any]]) -> Any: @@ -2304,6 +2309,7 @@ def invoke() -> Mapping[str, Any]: "upstream-moge": _load_moge, "pytorch-personaplex": _load_personaplex, "pytorch-timeseries": _load_timeseries, + "timm-classification": _load_vision, "upstream-fast-foundation-stereo": _load_fast_foundation_stereo, } diff --git a/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml b/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml new file mode 100644 index 0000000000..4de61a7d04 --- /dev/null +++ b/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: convnext-tiny-in12k-ft-in1k +candidate: + family: timm_convnext + checkpoint: timm/convnext_tiny.in12k_ft_in1k + revision: aa096f03029c7f0ec052013f64c819b34f8ad790 + task: classification + precision: fp16 + bundle: convnext-tiny-in12k-ft-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml b/families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml new file mode 100644 index 0000000000..0ec7fe66b8 --- /dev/null +++ b/families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: crossvit-9-240-in1k +candidate: + family: timm_crossvit + checkpoint: timm/crossvit_9_240.in1k + revision: df3051d20d8addd16d03c4230046128c17e32826 + task: classification + precision: fp16 + bundle: crossvit-9-240-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml new file mode 100644 index 0000000000..fce485638e --- /dev/null +++ b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: densenet121-ra-in1k +candidate: + family: timm_densenet + checkpoint: timm/densenet121.ra_in1k + task: classification + precision: fp16 + bundle: densenet121-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml b/families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml new file mode 100644 index 0000000000..05099a87ce --- /dev/null +++ b/families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dpn68b-ra-in1k +candidate: + family: timm_dpn + checkpoint: timm/dpn68b.ra_in1k + revision: fb49a617159bd2a0d6099742a6c57ef2d884dad9 + task: classification + precision: fp16 + bundle: dpn68b-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml b/families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml new file mode 100644 index 0000000000..e1ca938ee8 --- /dev/null +++ b/families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dpn92-mx-in1k +candidate: + family: timm_dpn + checkpoint: timm/dpn92.mx_in1k + revision: 8f2a71d2525254e14b0830a97250d841d5c4cae8 + task: classification + precision: fp16 + bundle: dpn92-mx-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml new file mode 100644 index 0000000000..3e552f7f45 --- /dev/null +++ b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: efficientnet-b0-ra-in1k +candidate: + family: timm_efficientnet + checkpoint: timm/efficientnet_b0.ra_in1k + task: classification + precision: fp16 + bundle: efficientnet-b0-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml b/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml new file mode 100644 index 0000000000..6b3f39f18a --- /dev/null +++ b/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: ghostnet-100-in1k +candidate: + family: timm_ghostnet + checkpoint: timm/ghostnet_100.in1k + revision: e524c2ee5d5a9412f802ce38ff895aba07b30910 + task: classification + precision: fp16 + bundle: ghostnet-100-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml b/families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml new file mode 100644 index 0000000000..92af0f0ee0 --- /dev/null +++ b/families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: hrnet-w18-ms-aug-in1k +candidate: + family: timm_hrnet + checkpoint: timm/hrnet_w18.ms_aug_in1k + revision: 7e2c5583769f54514fd87e3ba9de408e33eaba0f + task: classification + precision: fp16 + bundle: hrnet-w18-ms-aug-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml new file mode 100644 index 0000000000..f5e1ff44b7 --- /dev/null +++ b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: inception-v3-tv-in1k +candidate: + family: timm_inception + checkpoint: timm/inception_v3.tv_in1k + task: classification + precision: fp16 + bundle: inception-v3-tv-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml b/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml new file mode 100644 index 0000000000..0a3c2461b5 --- /dev/null +++ b/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: inception-resnet-v2-tf-in1k +candidate: + family: timm_inception_resnet + checkpoint: timm/inception_resnet_v2.tf_in1k + revision: 836b07d7d599da247de92f148c7a07fda18afced + task: classification + precision: fp16 + bundle: inception-resnet-v2-tf-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml b/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml new file mode 100644 index 0000000000..3dac070b5c --- /dev/null +++ b/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: inception-v4-tf-in1k +candidate: + family: timm_inception_v4 + checkpoint: timm/inception_v4.tf_in1k + revision: ad5d294cda312745a9afa433e45d9ccd956a1548 + task: classification + precision: fp16 + bundle: inception-v4-tf-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml b/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml new file mode 100644 index 0000000000..350285a356 --- /dev/null +++ b/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mnasnet-100-rmsp-in1k +candidate: + family: timm_mnasnet + checkpoint: timm/mnasnet_100.rmsp_in1k + task: classification + precision: fp16 + bundle: mnasnet-100-rmsp-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml new file mode 100644 index 0000000000..d22fec0ab6 --- /dev/null +++ b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv2-100-ra-in1k +candidate: + family: timm_mobilenetv2 + checkpoint: timm/mobilenetv2_100.ra_in1k + task: classification + precision: fp16 + bundle: mobilenetv2-100-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml new file mode 100644 index 0000000000..8a0a338665 --- /dev/null +++ b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv3-large-100-ra-in1k +candidate: + family: timm_mobilenetv3 + checkpoint: timm/mobilenetv3_large_100.ra_in1k + task: classification + precision: fp16 + bundle: mobilenetv3-large-100-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml new file mode 100644 index 0000000000..99cef8fa63 --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-large +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_large.e500_r256_in1k + task: classification + precision: fp16 + bundle: mobilenetv4-conv-large.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml new file mode 100644 index 0000000000..538206be01 --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-medium +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_medium.e500_r256_in1k + task: classification + precision: fp16 + bundle: mobilenetv4-conv-medium.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml new file mode 100644 index 0000000000..80a02ff7cc --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-small-050 +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_small_050.e3000_r224_in1k + task: classification + precision: fp16 + bundle: mobilenetv4-conv-small-050.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml new file mode 100644 index 0000000000..e9bcd35a24 --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-small +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_small.e2400_r224_in1k + task: classification + precision: fp16 + bundle: mobilenetv4-conv-small.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml b/families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml new file mode 100644 index 0000000000..ccc5de07f5 --- /dev/null +++ b/families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilevit-s-cvnets-in1k +candidate: + family: timm_mobilevit + checkpoint: timm/mobilevit_s.cvnets_in1k + revision: b531fe564ef88515b1aa1743ad4c40452b9f80ee + task: classification + precision: fp16 + bundle: mobilevit-s-cvnets-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml b/families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml new file mode 100644 index 0000000000..e3366f3447 --- /dev/null +++ b/families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilevit-xxs-cvnets-in1k +candidate: + family: timm_mobilevit + checkpoint: timm/mobilevit_xxs.cvnets_in1k + revision: 2c62d71504c6fbba9bb322f728586c4a235e5fa4 + task: classification + precision: fp16 + bundle: mobilevit-xxs-cvnets-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml b/families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml new file mode 100644 index 0000000000..ade48de911 --- /dev/null +++ b/families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dm-nfnet-f0-dm-in1k +candidate: + family: timm_nfnet + checkpoint: timm/dm_nfnet_f0.dm_in1k + revision: ca8e830961c2ebceffff7affff34acfe7a1da673 + task: classification + precision: fp16 + bundle: dm-nfnet-f0-dm-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml b/families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml new file mode 100644 index 0000000000..602e4776d4 --- /dev/null +++ b/families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dm-nfnet-f3-dm-in1k +candidate: + family: timm_nfnet + checkpoint: timm/dm_nfnet_f3.dm_in1k + revision: ca2451bf0778ae0665a2f13d1b2a7fb878c9ff27 + task: classification + precision: fp16 + bundle: dm-nfnet-f3-dm-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml b/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml new file mode 100644 index 0000000000..21b5b68cc1 --- /dev/null +++ b/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: regnety-040-ra3-in1k +candidate: + family: timm_regnet + checkpoint: timm/regnety_040.ra3_in1k + revision: a1875159c7b04f3b3189af804642eebba1e2c118 + task: classification + precision: fp16 + bundle: regnety-040-ra3-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml b/families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml new file mode 100644 index 0000000000..c211ea9b23 --- /dev/null +++ b/families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: repvgg-a2-rvgg-in1k +candidate: + family: timm_repvgg + checkpoint: timm/repvgg_a2.rvgg_in1k + revision: 87d4d383cb45031cb9fa2fc8ddca73fd6649240f + task: classification + precision: fp16 + bundle: repvgg-a2-rvgg-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml b/families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml new file mode 100644 index 0000000000..841441df62 --- /dev/null +++ b/families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: res2net50-26w-4s-in1k +candidate: + family: timm_res2net + checkpoint: timm/res2net50_26w_4s.in1k + revision: a36466d0354c9d337dbc0af329e13cb1cd7137fe + task: classification + precision: fp16 + bundle: res2net50-26w-4s-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml b/families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml new file mode 100644 index 0000000000..4d4d1ddd29 --- /dev/null +++ b/families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: res2net50d-in1k +candidate: + family: timm_res2net + checkpoint: timm/res2net50d.in1k + revision: 4f4bce4a1b5c17eabe5dde59af578ff9e1199ffc + task: classification + precision: fp16 + bundle: res2net50d-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_res2net/tests/benchmark/res2next50-in1k.yaml b/families/timm_res2net/tests/benchmark/res2next50-in1k.yaml new file mode 100644 index 0000000000..a3d1f3777c --- /dev/null +++ b/families/timm_res2net/tests/benchmark/res2next50-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: res2next50-in1k +candidate: + family: timm_res2net + checkpoint: timm/res2next50.in1k + revision: a8fe0f89e842b6dfa74b96deed2254d4fb7bf625 + task: classification + precision: fp16 + bundle: res2next50-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml b/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml new file mode 100644 index 0000000000..6b1a866328 --- /dev/null +++ b/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: resnest50d-in1k +candidate: + family: timm_resnest + checkpoint: timm/resnest50d.in1k + revision: 2239b0d822126dee3bc7a5357514de35b8f42092 + task: classification + precision: fp16 + bundle: resnest50d-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml new file mode 100644 index 0000000000..011118946e --- /dev/null +++ b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: resnet50-a1-in1k +candidate: + family: timm_resnet + checkpoint: timm/resnet50.a1_in1k + task: classification + precision: fp16 + bundle: resnet50-a1-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml b/families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml new file mode 100644 index 0000000000..31d4ed3f8e --- /dev/null +++ b/families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: senet154-gluon-in1k +candidate: + family: timm_senet + checkpoint: timm/senet154.gluon_in1k + revision: 86f6d239ed30acbe3aca0cbb5769da2621a4e02c + task: classification + precision: fp16 + bundle: senet154-gluon-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml b/families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml new file mode 100644 index 0000000000..6a3b0b1910 --- /dev/null +++ b/families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: seresnet50-a1-in1k +candidate: + family: timm_seresnet + checkpoint: timm/seresnet50.a1_in1k + revision: 23f3482ee9acc4f51a2668a7cdf255fbd9420417 + task: classification + precision: fp16 + bundle: seresnet50-a1-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml b/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml new file mode 100644 index 0000000000..def3334b85 --- /dev/null +++ b/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: swin-tiny-patch4-window7-224-ms-in1k +candidate: + family: timm_swin + checkpoint: timm/swin_tiny_patch4_window7_224.ms_in1k + revision: 4cc3a7275b50b53a7bec45f32c236ebe64227cff + task: classification + precision: fp16 + bundle: swin-tiny-patch4-window7-224-ms-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml b/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml new file mode 100644 index 0000000000..fd7368f3bf --- /dev/null +++ b/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: vgg16-tv-in1k +candidate: + family: timm_vgg + checkpoint: timm/vgg16.tv_in1k + task: classification + precision: fp16 + bundle: vgg16-tv-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml new file mode 100644 index 0000000000..57b53110a8 --- /dev/null +++ b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: timm-vit-base-p16-224-augreg-in21k-ft-in1k +candidate: + family: timm_vit + checkpoint: timm/vit_base_patch16_224.augreg_in21k_ft_in1k + task: classification + precision: fp16 + bundle: timm-vit-base-p16-224-augreg-in21k-ft-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml b/families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml new file mode 100644 index 0000000000..03fb8c4f41 --- /dev/null +++ b/families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xception41-tf-in1k +candidate: + family: timm_xception + checkpoint: timm/xception41.tf_in1k + revision: 8a17189361e63c972815ef62f2a30dd5b9f393b1 + task: classification + precision: fp16 + bundle: xception41-tf-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml b/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml new file mode 100644 index 0000000000..50150de079 --- /dev/null +++ b/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xcit-nano-12-p16-224-fb-in1k +candidate: + family: timm_xcit + checkpoint: timm/xcit_nano_12_p16_224.fb_in1k + revision: b181b4623913d78fd0927601c36e40b9d8bc74ce + task: classification + precision: fp16 + bundle: xcit-nano-12-p16-224-fb-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml b/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml new file mode 100644 index 0000000000..4d8d2be5c8 --- /dev/null +++ b/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xcit-tiny-12-p16-224-fb-in1k +candidate: + family: timm_xcit + checkpoint: timm/xcit_tiny_12_p16_224.fb_in1k + revision: 060449891838eaf301bdb3673e3daaf58e0370de + task: classification + precision: fp16 + bundle: xcit-tiny-12-p16-224-fb-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 269c67245e..88d895a377 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -38,6 +38,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _encoder_embedding_parity(case, context, dataset, output) elif metric_name == "forecast_tensor_parity": result = _etth1(case, context, definition, dataset, output) + elif metric_name == "image_classification_top1_parity": + result = _image_classification_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -701,6 +703,161 @@ def _candidate_outputs( return outputs, bundle +def _image_classification_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("image-classification dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError( + f"image-classification dataset requires at least {sample_limit} requests" + ) + + selected: list[dict[str, Any]] = [] + dataset_root = dataset.path.parent.resolve() + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"image-classification request {index} must be an object") + relative = request.get("image") + label = request.get("label") + if not isinstance(relative, str) or not relative: + raise QualificationError(f"image-classification request {index} has no image") + if isinstance(label, bool) or not isinstance(label, int) or label < 0: + raise QualificationError(f"image-classification request {index} has no valid label") + image = (dataset_root / relative).resolve() + if dataset_root not in image.parents or not image.is_file(): + raise QualificationError( + f"image-classification request {index} image is unavailable: {image}" + ) + selected.append( + { + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + "label": label, + } + ) + + reference = configured.get("reference", {}) + if not isinstance(reference, Mapping): + raise QualificationError("image-classification reference must be an object") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "batch_size": int(reference.get("batch_size", 16)), + "samples": selected, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + runner = ( + context.repository + / "tools/benchmark_qualification/references/timm_image_classification.py" + ) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"TIMM Accuracy reference failed; see {output}") + reference_payload = json.loads(reference_path.read_text(encoding="utf-8")) + expected = reference_payload.get("samples") + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("TIMM Accuracy reference returned an invalid sample set") + + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs( + case, context, output, "classify", candidate_requests + ) + rows = [] + for sample, reference_sample, candidate_sample in zip( + selected, expected, actual, strict=True + ): + reference_class = reference_sample.get("top_class") + candidate_class = candidate_sample.get("top_class") + if any( + isinstance(value, bool) or not isinstance(value, int) + for value in (reference_class, candidate_class) + ): + raise QualificationError("classification output omitted an integer top_class") + rows.append( + { + "sample_id": sample["sample_id"], + "label": sample["label"], + "reference_top_class": reference_class, + "candidate_top_class": candidate_class, + "reference_correct": reference_class == sample["label"], + "candidate_correct": candidate_class == sample["label"], + "passed": candidate_class == reference_class, + } + ) + + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("image-classification gate must be an object") + minimum_agreement = float(gate.get("min_top1_agreement", 0.98)) + maximum_drop = float(gate.get("max_top1_accuracy_drop_from_hf", 0.01)) + if not 0.0 <= minimum_agreement <= 1.0: + raise QualificationError("min_top1_agreement must be in [0, 1]") + if maximum_drop < 0.0 or not math.isfinite(maximum_drop): + raise QualificationError( + "max_top1_accuracy_drop_from_hf must be finite and nonnegative" + ) + agreement = sum(row["passed"] for row in rows) / len(rows) + reference_accuracy = sum(row["reference_correct"] for row in rows) / len(rows) + candidate_accuracy = sum(row["candidate_correct"] for row in rows) / len(rows) + passed = ( + agreement >= minimum_agreement + and candidate_accuracy >= reference_accuracy - maximum_drop + ) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "top1_agreement": agreement, + "reference_top1_accuracy": reference_accuracy, + "candidate_top1_accuracy": candidate_accuracy, + "top1_accuracy_drop_from_hf": reference_accuracy - candidate_accuracy, + }, + "gate": { + "min_top1_agreement": minimum_agreement, + "max_top1_accuracy_drop_from_hf": maximum_drop, + }, + "samples": rows, + } + + def _etth1_windows( path: Path, definition: Mapping[str, Any], diff --git a/tools/benchmark_qualification/benchmarks/image_classification_performance.yaml b/tools/benchmark_qualification/benchmarks/image_classification_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/image_classification_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/imagenette_classification.yaml b/tools/benchmark_qualification/benchmarks/imagenette_classification.yaml new file mode 100644 index 0000000000..d35af76dc9 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_classification.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: image_classification_top1_parity diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py index 4bfb96daa9..81acb638d9 100644 --- a/tools/benchmark_qualification/performance.py +++ b/tools/benchmark_qualification/performance.py @@ -7,6 +7,7 @@ import json import os +from pathlib import Path import sys from typing import Any, Mapping @@ -75,6 +76,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st assert isinstance(baseline, Mapping) assert isinstance(measurement, Mapping) assert isinstance(reference_timing, Mapping) + request = _resolve_family_assets(case, request) descriptor = write_model_descriptor(case, output, request) entry_id = f"qualification.{case.family}.{case.name}" suite = { @@ -217,3 +219,23 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st result["error"] = error write_result(output, result) return result + + +def _resolve_family_assets( + case: QualificationCase, request: Mapping[str, Any] +) -> dict[str, Any]: + resolved = dict(request) + family_root = case.source.parents[2].resolve() + for key, value in request.items(): + if not key.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if path.is_absolute(): + continue + path = (case.source.parent / path).resolve() + if family_root not in path.parents or not path.is_file(): + raise QualificationError( + f"Performance request asset {key!r} is unavailable inside {family_root}: {path}" + ) + resolved[key] = str(path) + return resolved diff --git a/tools/benchmark_qualification/references/timm_image_classification.py b/tools/benchmark_qualification/references/timm_image_classification.py new file mode 100644 index 0000000000..a917201a18 --- /dev/null +++ b/tools/benchmark_qualification/references/timm_image_classification.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Produce model-agnostic TIMM image-classification reference outputs.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Mapping, Sequence + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + value.add_argument("--request", required=True, type=Path) + value.add_argument("--output", required=True, type=Path) + return value + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + if not isinstance(request, Mapping): + raise ValueError("reference request must contain an object") + + import timm + import torch + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + model_id = _string(request.get("model"), "model") + revision = request.get("revision") + if revision is not None: + model_id += "@" + _string(revision, "revision") + dtype = { + "fp16": torch.float16, + "fp32": torch.float32, + "bf16": torch.bfloat16, + }[_string(request.get("precision"), "precision")] + batch_size = int(request.get("batch_size", 16)) + if batch_size < 1: + raise ValueError("batch_size must be positive") + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("samples must be a non-empty list") + + device = torch.device("cuda") + model = timm.create_model(f"hf-hub:{model_id}", pretrained=True) + model = model.eval().to(device=device, dtype=dtype) + transform = create_transform(**resolve_model_data_config(model), is_training=False) + outputs: list[dict[str, Any]] = [] + for start in range(0, len(samples), batch_size): + batch = samples[start : start + batch_size] + images = [] + for index, sample in enumerate(batch, start=start): + if not isinstance(sample, Mapping): + raise ValueError(f"sample {index} must be an object") + path = Path(_string(sample.get("image_path"), f"sample {index} image_path")) + with Image.open(path) as image: + images.append(transform(image.convert("RGB"))) + pixels = torch.stack(images).to(device=device, dtype=dtype) + with torch.inference_mode(): + classes = model(pixels).argmax(dim=-1).cpu().tolist() + outputs.extend( + { + "sample_id": _string(sample.get("sample_id"), "sample_id"), + "top_class": int(top_class), + } + for sample, top_class in zip(batch, classes, strict=True) + ) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": outputs}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} must be a non-empty string") + return value + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 493ca8bfd8..38806126cf 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -518,8 +518,10 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", "tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml", "tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", - "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", - "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", + "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", + "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", + "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", @@ -529,8 +531,9 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/catalog.py", "tools/benchmark_qualification/datasets.py", "tools/benchmark_qualification/performance.py", - "tools/benchmark_qualification/references/hf_encoder.py", - "tools/benchmark_qualification/references/hf_text_generation.py", + "tools/benchmark_qualification/references/hf_encoder.py", + "tools/benchmark_qualification/references/hf_text_generation.py", + "tools/benchmark_qualification/references/timm_image_classification.py", "tools/benchmark_qualification/runtime.py", "tools/ci/__init__.py", "tools/ci/__main__.py", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index fae6860f5b..9429ec4c45 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -13,7 +13,7 @@ import pytest -from apps.benchmark.performance.baselines import hf_transformers +from apps.benchmark.performance.baselines import hf_transformers, task_reference from apps.benchmark.performance.baselines.timing_contracts import timing_contract from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification import performance as qualification_performance @@ -240,6 +240,156 @@ def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) } +def test_image_classification_accuracy_compares_top1_and_gold_accuracy( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "Imagenette" + image_root = dataset_root / "images" + image_root.mkdir(parents=True) + for name in ("a.jpeg", "b.jpeg"): + (image_root / name).write_bytes(b"fixture") + dataset_path = dataset_root / "manifest.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + {"id": "a", "image": "images/a.jpeg", "label": 1}, + {"id": "b", "image": "images/b.jpeg", "label": 3}, + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/timm_example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + case = QualificationCase( + kind="accuracy", + model="example", + family="timm_example", + name="imagenette-parity", + benchmark="imagenette_classification", + candidate={ + "family": "timm_example", + "checkpoint": "timm/example", + "task": "classification", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 2, + "reference": {"precision": "fp32", "batch_size": 2}, + "gate": { + "min_top1_agreement": 1.0, + "max_top1_accuracy_drop_from_hf": 0.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("imagenette-validation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured: dict[str, object] = {} + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "a", "top_class": 1}, + {"sample_id": "b", "top_class": 2}, + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "image_classification_top1_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"top_class": 1}, {"top_class": 2}], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"] == { + "samples": 2, + "top1_agreement": 1.0, + "reference_top1_accuracy": 0.5, + "candidate_top1_accuracy": 0.5, + "top1_accuracy_drop_from_hf": 0.0, + } + assert captured["batch_size"] == 2 + assert captured["samples"][0]["image_path"] == str(image_root / "a.jpeg") + + +def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> None: + profile = tmp_path / "families/timm_example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + image = profile.parent.parent / "data/test.jpeg" + image.parent.mkdir(parents=True) + image.write_bytes(b"fixture") + case = QualificationCase( + kind="performance", + model="example", + family="timm_example", + name="classify", + benchmark="image_classification_performance", + candidate={ + "family": "timm_example", + "checkpoint": "timm/example", + "task": "classification", + "precision": "fp16", + "build": {}, + }, + values={}, + source=profile, + reference_requirements=None, + ) + + resolved = qualification_performance._resolve_family_assets( + case, {"image_path": "../data/test.jpeg", "batch_size": 1} + ) + + assert resolved == {"image_path": str(image.resolve()), "batch_size": 1} + + +def test_timm_classification_uses_a_generic_task_reference_adapter() -> None: + assert "timm-classification" in task_reference.ADAPTERS + assert task_reference.LOADERS["timm-classification"] is task_reference._load_vision + + def test_accuracy_forwards_seq2seq_reference_contract_and_nested_dataset_input( tmp_path: Path, monkeypatch ) -> None: From edbe9861fc3e66859fafeacd44f60012b60af663 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 18:10:11 +0000 Subject: [PATCH 031/100] feat(benchmark): qualify speech recognition Add family-owned Accuracy references and automatically discovered profiles for seven batch and streaming ASR models. Measure LibriSpeech WER against labeled data and each official reference while exercising the public transcribe operation for Performance. Signed-off-by: chaofengw --- .../performance/baselines/task_reference.py | 46 +++- .../canary/tests/benchmark/canary-1b-v2.yaml | 47 ++++ families/canary/tests/benchmark/reference.py | 45 ++++ families/glmasr/requirements.txt | 6 + .../tests/benchmark/glmasr-nano-fp16.yaml | 48 ++++ families/glmasr/tests/benchmark/reference.py | 59 +++++ .../nemotron-3.5-asr-streaming-0.6b.yaml | 52 +++++ .../nemotron-speech-streaming-en-0.6b.yaml | 49 ++++ .../tests/benchmark/reference.py | 85 +++++++ families/whisper/requirements.txt | 4 +- families/whisper/tests/benchmark/reference.py | 59 +++++ .../benchmark/whisper-large-v3-turbo.yaml | 48 ++++ .../tests/benchmark/whisper-small-fp16.yaml | 50 ++++ .../tests/benchmark/whisper-tiny-fp16.yaml | 49 ++++ tools/benchmark_qualification/accuracy.py | 216 ++++++++++++++++++ .../benchmarks/librispeech_transcription.yaml | 20 ++ .../benchmarks/speech_nemo_performance.yaml | 15 ++ .../speech_transformer_performance.yaml | 15 ++ .../references/speech_io.py | 113 +++++++++ tools/tests/test_architecture.py | 8 +- tools/tests/test_model_benchmark.py | 138 +++++++++++ 21 files changed, 1168 insertions(+), 4 deletions(-) create mode 100644 families/canary/tests/benchmark/canary-1b-v2.yaml create mode 100644 families/canary/tests/benchmark/reference.py create mode 100644 families/glmasr/requirements.txt create mode 100644 families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml create mode 100644 families/glmasr/tests/benchmark/reference.py create mode 100644 families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml create mode 100644 families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml create mode 100644 families/nemotron_speech_streaming/tests/benchmark/reference.py create mode 100644 families/whisper/tests/benchmark/reference.py create mode 100644 families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml create mode 100644 families/whisper/tests/benchmark/whisper-small-fp16.yaml create mode 100644 families/whisper/tests/benchmark/whisper-tiny-fp16.yaml create mode 100644 tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml create mode 100644 tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml create mode 100644 tools/benchmark_qualification/references/speech_io.py diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index d008a70825..a51b6080a9 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -67,6 +67,7 @@ ) ADAPTERS = ( "hf-diffusers", + "hf-chat-asr", "hf-qwen3-omni", "hf-transformers-asr", "hf-transformers-embedding", @@ -550,7 +551,49 @@ def _load_asr( max_new_tokens = int(request.get("max_new_tokens", 100)) device = torch.device("cuda") - if arguments.family in {"canary", "nemotron_speech_streaming"}: + if arguments.adapter == "hf-chat-asr": + from transformers import AutoModel, AutoProcessor + + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + AutoModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + conversation = [ + { + "role": "user", + "content": [ + {"type": "audio", "audio": audio}, + {"type": "text", "text": "Please transcribe this audio into text"}, + ], + } + ] + inputs = processor.apply_chat_template( + conversation, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, + sampling_rate=target_rate, + ).to(model.device) + prompt_tokens = int(inputs["input_ids"].shape[1]) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + generated = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + ) + token_ids = [int(token) for token in generated[0, prompt_tokens:].cpu().tolist()] + return { + "text": processor.tokenizer.decode(token_ids, skip_special_tokens=True).strip(), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + elif arguments.family in {"canary", "nemotron_speech_streaming"}: model = _load_nemo_asr_reference_model(arguments, device=device).eval().to(device) _disable_nemo_asr_cuda_graphs(model) reference_dtype = _torch_dtype(torch, arguments.precision) @@ -2296,6 +2339,7 @@ def invoke() -> Mapping[str, Any]: str, Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session] ] = { "hf-diffusers": _load_diffusers, + "hf-chat-asr": _load_asr, "hf-qwen3-omni": _load_qwen3_omni, "hf-transformers-asr": _load_asr, "hf-transformers-embedding": _load_embedding, diff --git a/families/canary/tests/benchmark/canary-1b-v2.yaml b/families/canary/tests/benchmark/canary-1b-v2.yaml new file mode 100644 index 0000000000..a0d5521132 --- /dev/null +++ b/families/canary/tests/benchmark/canary-1b-v2.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: canary-1b-v2 +candidate: + family: canary + checkpoint: nvidia/canary-1b-v2 + task: transcription + precision: fp16 + bundle: canary-1b-v2.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 50 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.3 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_nemo_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: nemo-asr + mode: pytorch-eager + reference_backend: nemo_reference + precision: fp32 + output_contract: transcription-text diff --git a/families/canary/tests/benchmark/reference.py b/families/canary/tests/benchmark/reference.py new file mode 100644 index 0000000000..2c183bb216 --- /dev/null +++ b/families/canary/tests/benchmark/reference.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned NeMo Canary reference for Accuracy qualification.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Sequence + +from tools.benchmark_qualification.references import speech_io + + +def _text(value: Any) -> str: + return str(value.text if hasattr(value, "text") else value) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + from huggingface_hub import snapshot_download + from nemo.collections.asr.models import ASRModel + + model_id, revision = speech_io.model_id(request) + snapshot = snapshot_download( + repo_id=model_id, + revision=revision, + allow_patterns=["*.nemo"], + ) + archives = sorted(Path(snapshot).glob("*.nemo")) + if not archives: + raise FileNotFoundError(f"Canary NeMo archive is missing for {model_id}") + model = ASRModel.restore_from(str(archives[0]), map_location="cpu").cpu().eval() + values = model.transcribe([str(sample.wav_path) for sample in samples], batch_size=1) + if isinstance(values, tuple): + values = values[0] + speech_io.write_result(arguments.output, samples, [_text(value) for value in values]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/glmasr/requirements.txt b/families/glmasr/requirements.txt new file mode 100644 index 0000000000..be9ba894f6 --- /dev/null +++ b/families/glmasr/requirements.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +scipy==1.15.3 +soundfile==0.14.0 +transformers==5.2.0 diff --git a/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml b/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml new file mode 100644 index 0000000000..d840cf93c3 --- /dev/null +++ b/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: glmasr-nano-fp16 +candidate: + family: glmasr + checkpoint: zai-org/GLM-ASR-Nano-2512 + revision: 61ba4e0b3309b6656edea3e93e419f7bd5c61957 + task: transcription + precision: fp16 + bundle: glmasr-nano-fp16.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 72 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/librispeech-test-clean-6930-75918-0003.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-chat-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/glmasr/tests/benchmark/reference.py b/families/glmasr/tests/benchmark/reference.py new file mode 100644 index 0000000000..d79c83474f --- /dev/null +++ b/families/glmasr/tests/benchmark/reference.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official GLM-ASR reference for Accuracy qualification.""" + +from __future__ import annotations + +from typing import Sequence + +from tools.benchmark_qualification.references import speech_io + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + import torch + from transformers import GlmAsrForConditionalGeneration, GlmAsrProcessor + + model_id, revision = speech_io.model_id(request) + options = {"revision": revision} if revision else {} + processor = GlmAsrProcessor.from_pretrained(model_id, **options) + model = GlmAsrForConditionalGeneration.from_pretrained( + model_id, + torch_dtype=speech_io.torch_dtype(torch, request), + **options, + ).eval().to("cuda") + max_new_tokens = int(request.get("max_new_tokens", 128)) + texts = [] + for sample in samples: + conversation = [ + { + "role": "user", + "content": [ + {"type": "audio", "audio": sample.waveform}, + {"type": "text", "text": "Please transcribe this audio into text"}, + ], + } + ] + inputs = processor.apply_chat_template( + conversation, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, + sampling_rate=sample.sample_rate, + ).to(model.device) + with torch.inference_mode(): + generated = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) + new_tokens = generated[0, inputs["input_ids"].shape[1] :] + texts.append(processor.tokenizer.decode(new_tokens, skip_special_tokens=True)) + speech_io.write_result(arguments.output, samples, texts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml new file mode 100644 index 0000000000..eafdd865cd --- /dev/null +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-3.5-asr-streaming-0.6b +candidate: + family: nemotron_speech_streaming + checkpoint: nvidia/nemotron-3.5-asr-streaming-0.6b + task: transcription_streaming + precision: fp16 + bundle: nemotron-3.5-asr-streaming-0.6b.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 80 + language: en-US + streaming: true + chunk_ms: 1120 + att_context_left: 56 + att_context_right: 13 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.35 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe-streaming + benchmark: speech_nemo_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: nemo-asr + mode: pytorch-eager + reference_backend: nemo_reference + precision: fp32 + output_contract: transcription-text diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml new file mode 100644 index 0000000000..4695f1c80d --- /dev/null +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-speech-streaming-en-0.6b +candidate: + family: nemotron_speech_streaming + checkpoint: nvidia/nemotron-speech-streaming-en-0.6b + task: transcription_streaming + precision: fp16 + bundle: nemotron-speech-streaming-en-0.6b.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 80 + streaming: true + chunk_ms: 1120 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.35 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe-streaming + benchmark: speech_nemo_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: nemo-asr + mode: pytorch-eager + reference_backend: nemo_reference + precision: fp32 + output_contract: transcription-text diff --git a/families/nemotron_speech_streaming/tests/benchmark/reference.py b/families/nemotron_speech_streaming/tests/benchmark/reference.py new file mode 100644 index 0000000000..56e40841b2 --- /dev/null +++ b/families/nemotron_speech_streaming/tests/benchmark/reference.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned NeMo streaming-ASR reference for Accuracy qualification.""" + +from __future__ import annotations + +import json +from typing import Any, Sequence + +from tools.benchmark_qualification.references import speech_io + + +def _text(value: Any) -> str: + return str(value.text if hasattr(value, "text") else value) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + import torch + from nemo.collections.asr.models import ASRModel + + model_id, revision = speech_io.model_id(request) + is_nemotron35 = "nemotron-3.5-asr-streaming" in model_id.casefold() + if is_nemotron35: + from apps.benchmark.performance.baselines.audio_reference import ( + load_nemotron35_asr_model, + ) + + model = load_nemotron35_asr_model( + model=model_id, + revision=revision or "", + local_files_only=False, + device="cuda", + ) + else: + model = ASRModel.from_pretrained(model_id, map_location="cpu").eval().to("cuda") + + language = request.get("language") + manifest = arguments.output.parent / "reference-audio" / "manifest.jsonl" + records = [] + for sample in samples: + value = { + "audio_filepath": str(sample.wav_path), + "duration": float(len(sample.waveform)) / sample.sample_rate, + "text": "", + } + if isinstance(language, str) and language: + value["lang"] = language + records.append(value) + manifest.write_text( + "".join(json.dumps(value, ensure_ascii=False) + "\n" for value in records), + encoding="utf-8", + ) + + original_forward = model.forward + if is_nemotron35: + + def forward_with_extended_prompt(*args: Any, **kwargs: Any) -> Any: + prompt = kwargs.get("prompt") + if prompt is not None and prompt.shape[1] > 0: + kwargs = dict(kwargs) + kwargs["prompt"] = torch.cat((prompt, prompt[:, -1:, :]), dim=1) + return original_forward(*args, **kwargs) + + model.forward = forward_with_extended_prompt + try: + options = {"batch_size": 1} + if is_nemotron35: + options["verbose"] = False + values = model.transcribe(str(manifest), **options) + finally: + model.forward = original_forward + if isinstance(values, tuple): + values = values[0] + speech_io.write_result(arguments.output, samples, [_text(value) for value in values]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/whisper/requirements.txt b/families/whisper/requirements.txt index 860aa73c19..be9ba894f6 100644 --- a/families/whisper/requirements.txt +++ b/families/whisper/requirements.txt @@ -1,4 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -scipy +scipy==1.15.3 +soundfile==0.14.0 +transformers==5.2.0 diff --git a/families/whisper/tests/benchmark/reference.py b/families/whisper/tests/benchmark/reference.py new file mode 100644 index 0000000000..bcc3d0aef4 --- /dev/null +++ b/families/whisper/tests/benchmark/reference.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers Whisper reference for Accuracy qualification.""" + +from __future__ import annotations + +from typing import Sequence + +from tools.benchmark_qualification.references import speech_io + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + import torch + from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + + model_id, revision = speech_io.model_id(request) + options = {"revision": revision} if revision else {} + processor = AutoProcessor.from_pretrained(model_id, **options) + model = AutoModelForSpeechSeq2Seq.from_pretrained( + model_id, + torch_dtype=speech_io.torch_dtype(torch, request), + **options, + ).eval().to("cuda") + max_new_tokens = int(request.get("max_new_tokens", 128)) + language = request.get("language") + texts = [] + for sample in samples: + inputs = processor( + sample.waveform, + sampling_rate=sample.sample_rate, + return_tensors="pt", + ) + model_dtype = next(model.parameters()).dtype + inputs = { + key: ( + value.to(device=model.device, dtype=model_dtype) + if value.is_floating_point() + else value.to(device=model.device) + ) + for key, value in inputs.items() + } + generation = {"max_new_tokens": max_new_tokens} + if isinstance(language, str) and language: + generation["language"] = language + with torch.inference_mode(): + token_ids = model.generate(**inputs, **generation) + texts.append(processor.batch_decode(token_ids, skip_special_tokens=True)[0]) + speech_io.write_result(arguments.output, samples, texts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml b/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml new file mode 100644 index 0000000000..4bd54f5cd2 --- /dev/null +++ b/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: whisper-large-v3-turbo +candidate: + family: whisper + checkpoint: openai/whisper-large-v3-turbo + task: transcription + precision: fp32 + bundle: whisper-large-v3-turbo.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 120 + language: en + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/whisper/tests/benchmark/whisper-small-fp16.yaml b/families/whisper/tests/benchmark/whisper-small-fp16.yaml new file mode 100644 index 0000000000..3777e23685 --- /dev/null +++ b/families/whisper/tests/benchmark/whisper-small-fp16.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: whisper-small-fp16 +candidate: + family: whisper + checkpoint: openai/whisper-small + revision: 973afd24965f72e36ca33b3055d56a652f456b4d + task: transcription + precision: fp16 + bundle: whisper-small-fp16.bundle + build: + max_sequence_length: 128 + fp32_layers: [0] + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 120 + language: en + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml b/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml new file mode 100644 index 0000000000..f0306c82eb --- /dev/null +++ b/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: whisper-tiny-fp16 +candidate: + family: whisper + checkpoint: openai/whisper-tiny + task: transcription + precision: fp16 + bundle: whisper-tiny-fp16.bundle + build: + max_sequence_length: 128 + fp32_layers: [0] + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 120 + language: en + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/librispeech-test-clean-6930-75918-0003.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 88d895a377..b4ba353b62 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -9,6 +9,7 @@ import json import math import random +import re from pathlib import Path from typing import Any, Mapping, Sequence @@ -40,6 +41,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _etth1(case, context, definition, dataset, output) elif metric_name == "image_classification_top1_parity": result = _image_classification_parity(case, context, dataset, output) + elif metric_name == "speech_transcription_wer_parity": + result = _speech_transcription_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -858,6 +861,219 @@ def _image_classification_parity( } +def _speech_transcription_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"speech dataset requires at least {sample_limit} requests") + + selected = [] + data_root = context.data_root.resolve() + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"speech request {index} must be an object") + audio_value = _speech_audio(request) + audio = (data_root / audio_value).resolve() + if data_root not in audio.parents or not audio.is_file(): + fallback = (dataset.path.parent / audio_value).resolve() + if data_root not in fallback.parents or not fallback.is_file(): + raise QualificationError(f"speech request {index} audio is unavailable: {audio}") + audio = fallback + gold = request.get("reference") + if not isinstance(gold, str) or not gold.strip(): + raise QualificationError(f"speech request {index} has no reference transcript") + selected.append( + { + "sample_id": str(request.get("id") or f"sample-{index}"), + "audio_path": str(audio), + "gold_text": gold, + } + ) + + reference = configured.get("reference") + candidate_request = configured.get("request", {}) + if not isinstance(reference, Mapping) or not isinstance(candidate_request, Mapping): + raise QualificationError("speech Accuracy reference and request must be objects") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("speech Accuracy reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"speech reference runner is not family-owned: {runner}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "max_new_tokens": int(candidate_request.get("max_new_tokens", 128)), + "language": candidate_request.get("language"), + "samples": selected, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"speech Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("speech reference returned an invalid sample set") + + candidate_requests = [] + for sample in expected: + if not isinstance(sample, Mapping): + raise QualificationError("speech reference sample must be an object") + audio_path = sample.get("audio_path") + if not isinstance(audio_path, str) or not Path(audio_path).is_file(): + raise QualificationError("speech reference did not produce candidate WAV audio") + candidate_requests.append( + { + "sample_id": str(sample.get("sample_id", "")), + "request": {**dict(candidate_request), "audio_path": audio_path}, + } + ) + actual, bundle = _candidate_outputs( + case, context, output, "transcribe", candidate_requests + ) + + rows = [] + gold_reference_counts = [0, 0] + gold_candidate_counts = [0, 0] + parity_counts = [0, 0] + for source, reference_sample, candidate_sample in zip( + selected, expected, actual, strict=True + ): + reference_text = reference_sample.get("text") + candidate_text = candidate_sample.get("text") + if not isinstance(reference_text, str) or not reference_text.strip(): + raise QualificationError("speech reference returned an empty transcript") + if not isinstance(candidate_text, str): + raise QualificationError("speech candidate omitted its transcript") + reference_gold = _word_error_counts(source["gold_text"], reference_text) + candidate_gold = _word_error_counts(source["gold_text"], candidate_text) + candidate_reference = _word_error_counts(reference_text, candidate_text) + for totals, counts in ( + (gold_reference_counts, reference_gold), + (gold_candidate_counts, candidate_gold), + (parity_counts, candidate_reference), + ): + totals[0] += counts[0] + totals[1] += counts[1] + rows.append( + { + "sample_id": source["sample_id"], + "reference_text": reference_text, + "candidate_text": candidate_text, + "gold_text": source["gold_text"], + "reference_wer": _rate(reference_gold), + "candidate_wer": _rate(candidate_gold), + "wer_to_reference": _rate(candidate_reference), + } + ) + + reference_wer = _rate(gold_reference_counts) + candidate_wer = _rate(gold_candidate_counts) + parity_wer = _rate(parity_counts) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("speech Accuracy gate must be an object") + parity_limit = float(gate.get("max_wer_to_reference", 0.1)) + increase_limit = float(gate.get("max_wer_increase_from_reference", 0.02)) + if any(not math.isfinite(value) or value < 0.0 for value in (parity_limit, increase_limit)): + raise QualificationError("speech WER gates must be finite and nonnegative") + increase = candidate_wer - reference_wer + passed = parity_wer <= parity_limit and increase <= increase_limit + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "reference_wer": reference_wer, + "candidate_wer": candidate_wer, + "wer_increase_from_reference": increase, + "wer_to_reference": parity_wer, + }, + "gate": { + "max_wer_to_reference": parity_limit, + "max_wer_increase_from_reference": increase_limit, + }, + "samples": rows, + } + + +def _speech_audio(request: Mapping[str, Any]) -> str: + messages = request.get("messages") + if not isinstance(messages, list): + raise QualificationError("speech request has no messages") + for message in messages: + if not isinstance(message, Mapping): + continue + content = message.get("content") + if not isinstance(content, list): + continue + for item in content: + if isinstance(item, Mapping) and item.get("type") == "audio": + audio = item.get("audio") + if isinstance(audio, str) and audio: + return audio + raise QualificationError("speech request has no audio content") + + +def _word_error_counts(reference: str, hypothesis: str) -> list[int]: + def words(value: str) -> list[str]: + return [ + normalized + for word in value.split() + if (normalized := re.sub(r"^[^\w]+|[^\w]+$", "", word).casefold()) + ] + + left = words(reference) + right = words(hypothesis) + previous = list(range(len(right) + 1)) + for row, left_word in enumerate(left, start=1): + current = [row] + for column, right_word in enumerate(right, start=1): + current.append( + min( + current[-1] + 1, + previous[column] + 1, + previous[column - 1] + (left_word != right_word), + ) + ) + previous = current + return [previous[-1], len(left)] + + +def _rate(counts: Sequence[int]) -> float: + return float(counts[0]) / max(int(counts[1]), 1) + + def _etth1_windows( path: Path, definition: Mapping[str, Any], diff --git a/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml b/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml new file mode 100644 index 0000000000..f9ce441811 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: librispeech-clean-test + version: test-clean-task-eval + sha256: 45306828857c7e4174baa52dab089214a8b2956165dacad3eaec8d2aa1483b1f + source: + mode: manual + path: librispeech_clean_test/librispeech_clean_test.json + instructions: https://www.openslr.org/12 + license: CC-BY-4.0 +selection: + method: first +input: + adapter: audio_path +metric: + name: speech_transcription_wer_parity diff --git a/tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml b/tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml new file mode 100644 index 0000000000..d12095f992 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: true +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml b/tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/references/speech_io.py b/tools/benchmark_qualification/references/speech_io.py new file mode 100644 index 0000000000..b662f4d9d1 --- /dev/null +++ b/tools/benchmark_qualification/references/speech_io.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic request and PCM mechanics for family-owned ASR references.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import math +from pathlib import Path +from typing import Any, Mapping, Sequence + + +@dataclass(frozen=True) +class SpeechSample: + sample_id: str + gold_text: str + waveform: Any + sample_rate: int + wav_path: Path + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser() + value.add_argument("--request", required=True, type=Path) + value.add_argument("--output", required=True, type=Path) + return value + + +def load_request(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("speech reference request must be an object") + return value + + +def prepare_samples(request: Mapping[str, Any], output: Path) -> list[SpeechSample]: + import numpy as np + from scipy.signal import resample_poly + import soundfile as sf + + configured = request.get("samples") + if not isinstance(configured, list) or not configured: + raise ValueError("speech reference requires a non-empty samples list") + audio_root = output.parent / "reference-audio" + audio_root.mkdir(parents=True, exist_ok=True) + prepared = [] + for index, value in enumerate(configured): + if not isinstance(value, Mapping): + raise ValueError(f"speech reference sample {index} must be an object") + sample_id = required_string(value.get("sample_id"), "sample_id") + gold_text = required_string(value.get("gold_text"), "gold_text") + source = Path(required_string(value.get("audio_path"), "audio_path")) + if not source.is_file(): + raise FileNotFoundError(source) + waveform, sample_rate = sf.read(source, dtype="float32", always_2d=True) + waveform = waveform.mean(axis=1) + target_rate = 16_000 + if int(sample_rate) != target_rate: + divisor = math.gcd(int(sample_rate), target_rate) + waveform = resample_poly( + waveform, target_rate // divisor, int(sample_rate) // divisor + ).astype(np.float32) + wav_path = audio_root / f"{index:04d}.wav" + sf.write(wav_path, waveform, target_rate, subtype="PCM_16") + prepared.append(SpeechSample(sample_id, gold_text, waveform, target_rate, wav_path)) + return prepared + + +def write_result(output: Path, samples: Sequence[SpeechSample], texts: Sequence[str]) -> None: + if len(samples) != len(texts): + raise ValueError("speech reference returned a different sample count") + rows = [] + for sample, text in zip(samples, texts, strict=True): + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"speech reference returned an empty transcript for {sample.sample_id}") + rows.append( + { + "sample_id": sample.sample_id, + "text": text.strip(), + "audio_path": str(sample.wav_path), + } + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps({"samples": rows}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def required_string(value: Any, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + return value + + +def model_id(request: Mapping[str, Any]) -> tuple[str, str | None]: + model = required_string(request.get("model"), "model") + revision = request.get("revision") + if revision is not None and (not isinstance(revision, str) or not revision): + raise ValueError("revision must be a non-empty string when set") + return model, revision + + +def torch_dtype(torch_module: Any, request: Mapping[str, Any]) -> Any: + return { + "fp16": torch_module.float16, + "fp32": torch_module.float32, + "bf16": torch_module.bfloat16, + }[required_string(request.get("precision"), "precision")] diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 38806126cf..96c720bf17 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -521,10 +521,13 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", + "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", - "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", - "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", + "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", + "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", + "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", + "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", "tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml", @@ -533,6 +536,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/performance.py", "tools/benchmark_qualification/references/hf_encoder.py", "tools/benchmark_qualification/references/hf_text_generation.py", + "tools/benchmark_qualification/references/speech_io.py", "tools/benchmark_qualification/references/timm_image_classification.py", "tools/benchmark_qualification/runtime.py", "tools/ci/__init__.py", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 9429ec4c45..5a2d30f394 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -390,6 +390,144 @@ def test_timm_classification_uses_a_generic_task_reference_adapter() -> None: assert task_reference.LOADERS["timm-classification"] is task_reference._load_vision +def test_chat_asr_uses_the_generic_asr_reference_loader() -> None: + assert "hf-chat-asr" in task_reference.ADAPTERS + assert task_reference.LOADERS["hf-chat-asr"] is task_reference._load_asr + + +def test_speech_accuracy_reports_reference_and_labeled_wer( + tmp_path: Path, monkeypatch +) -> None: + data_root = tmp_path / "datasets" + audio_root = data_root / "speech/audio" + audio_root.mkdir(parents=True) + for name in ("a.flac", "b.flac"): + (audio_root / name).write_bytes(b"fixture") + dataset_path = data_root / "speech/dataset.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "id": "a", + "messages": [ + { + "role": "user", + "content": [{"type": "audio", "audio": "speech/audio/a.flac"}], + } + ], + "reference": "hello world", + }, + { + "id": "b", + "messages": [ + { + "role": "user", + "content": [{"type": "audio", "audio": "speech/audio/b.flac"}], + } + ], + "reference": "one two", + }, + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="example", + name="librispeech-wer", + benchmark="librispeech_transcription", + candidate={ + "family": "example", + "checkpoint": "org/example", + "task": "transcription", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 2, + "request": {"max_new_tokens": 32}, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "max_wer_to_reference": 0.3, + "max_wer_increase_from_reference": 0.3, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("speech", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=data_root, + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + + def reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + wavs = [tmp_path / "a.wav", tmp_path / "b.wav"] + for wav in wavs: + wav.write_bytes(b"fixture") + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "a", "text": "hello world", "audio_path": str(wavs[0])}, + {"sample_id": "b", "text": "one two", "audio_path": str(wavs[1])}, + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "speech_transcription_wer_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"text": "hello world"}, {"text": "one three"}], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"] == { + "samples": 2, + "reference_wer": 0.0, + "candidate_wer": 0.25, + "wer_increase_from_reference": 0.25, + "wer_to_reference": 0.25, + } + + def test_accuracy_forwards_seq2seq_reference_contract_and_nested_dataset_input( tmp_path: Path, monkeypatch ) -> None: From 88b3f2b55eb601a22bf929e041bb53ee0279150b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 18:27:06 +0000 Subject: [PATCH 032/100] feat(benchmark): qualify DINOv3 image features Signed-off-by: chaofengw --- ...inov3-convnext-tiny-pretrain-lvd1689m.yaml | 48 +++ .../dinov3-vits16-pretrain-lvd1689m.yaml | 48 +++ families/dinov3/tests/benchmark/reference.py | 78 ++++ tools/benchmark_qualification/accuracy.py | 347 +++++++++++++++--- .../benchmarks/beans_image_feature_knn.yaml | 20 + .../image_features_performance.yaml | 15 + tools/tests/test_architecture.py | 2 + tools/tests/test_model_benchmark.py | 169 ++++++++- 8 files changed, 661 insertions(+), 66 deletions(-) create mode 100644 families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml create mode 100644 families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml create mode 100644 families/dinov3/tests/benchmark/reference.py create mode 100644 tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml create mode 100644 tools/benchmark_qualification/benchmarks/image_features_performance.yaml diff --git a/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml new file mode 100644 index 0000000000..b8b69a46ac --- /dev/null +++ b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dinov3-convnext-tiny-pretrain-lvd1689m +candidate: + family: dinov3 + checkpoint: facebook/dinov3-convnext-tiny-pretrain-lvd1689m + revision: 10d30274b4d445111e2d5bf75ac93bbd94db274b + task: image_features + precision: fp16 + bundle: dinov3-convnext-tiny-pretrain-lvd1689m.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: beans-knn-parity + benchmark: beans_image_feature_knn + bank_samples_per_class: 24 + query_samples_per_class: 16 + knn_k: 10 + knn_temperature: 0.07 + reference: + command: reference.py + precision: fp32 + batch_size: 16 + gate: + min_pooler_cosine: 0.999 + min_vector_pass_rate: 1.0 + min_knn_top1_agreement: 0.98 + max_knn_accuracy_drop_from_hf: 0.02 + +performance: + - name: extract-features + benchmark: image_features_performance + operation: extract_features + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-vision + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: image-features-shape diff --git a/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml new file mode 100644 index 0000000000..8fd63e347f --- /dev/null +++ b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dinov3-vits16-pretrain-lvd1689m +candidate: + family: dinov3 + checkpoint: facebook/dinov3-vits16-pretrain-lvd1689m + revision: 114c1379950215c8b35dfcd4e90a5c251dde0d32 + task: image_features + precision: fp16 + bundle: dinov3-vits16-pretrain-lvd1689m.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: beans-knn-parity + benchmark: beans_image_feature_knn + bank_samples_per_class: 24 + query_samples_per_class: 16 + knn_k: 10 + knn_temperature: 0.07 + reference: + command: reference.py + precision: fp32 + batch_size: 16 + gate: + min_pooler_cosine: 0.999 + min_vector_pass_rate: 1.0 + min_knn_top1_agreement: 0.98 + max_knn_accuracy_drop_from_hf: 0.02 + +performance: + - name: extract-features + benchmark: image_features_performance + operation: extract_features + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-vision + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: image-features-shape diff --git a/families/dinov3/tests/benchmark/reference.py b/families/dinov3/tests/benchmark/reference.py new file mode 100644 index 0000000000..469e2ee307 --- /dev/null +++ b/families/dinov3/tests/benchmark/reference.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers DINOv3 reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Sequence + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModel + + model_id = str(request["model"]) + revision = request.get("revision") + options = {"revision": revision} if revision else {} + processor = AutoImageProcessor.from_pretrained(model_id, **options) + model = ( + AutoModel.from_pretrained( + model_id, torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), **options + ) + .eval() + .to("cuda") + ) + batch_size = int(request.get("batch_size", 16)) + result: list[dict[str, Any]] = [] + for start in range(0, len(samples), batch_size): + batch = samples[start : start + batch_size] + images = [Image.open(str(sample["image_path"])).convert("RGB") for sample in batch] + inputs = processor(images=images, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + with torch.inference_mode(): + pooled = model(**inputs).pooler_output.float().cpu() + for sample, vector in zip(batch, pooled, strict=True): + result.append( + { + "sample_id": str(sample["sample_id"]), + "pooler_output": vector.tolist(), + } + ) + arguments.output.write_text(json.dumps({"samples": result}, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index b4ba353b62..dabdab79c7 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -43,6 +43,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _image_classification_parity(case, context, dataset, output) elif metric_name == "speech_transcription_wer_parity": result = _speech_transcription_parity(case, context, dataset, output) + elif metric_name == "image_feature_knn_parity": + result = _image_feature_knn_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -67,7 +69,9 @@ def _text_generation_parity( raise QualificationError(f"text-generation request {index} must be an object") selected.append( { - "sample_id": str(request.get("id") or request.get("sample_id") or f"sample-{index}"), + "sample_id": str( + request.get("id") or request.get("sample_id") or f"sample-{index}" + ), "prompt": _prompt(request), } ) @@ -208,10 +212,7 @@ def _encoder_embedding_parity( ) from error model_class = reference.get("model_class", "auto") tokenizer_class = reference.get("tokenizer_class", "auto") - if not all( - isinstance(value, str) and value - for value in (model_class, tokenizer_class) - ): + if not all(isinstance(value, str) and value for value in (model_class, tokenizer_class)): raise QualificationError("encoder reference classes must be non-empty strings") reference_request = { "model": str(case.candidate["checkpoint"]), @@ -255,9 +256,7 @@ def _encoder_embedding_parity( } for sample in samples ] - candidate, bundle = _candidate_outputs( - case, context, output, operation, candidate_requests - ) + candidate, bundle = _candidate_outputs(case, context, output, operation, candidate_requests) compared = _compare_encoder_embeddings(reference_samples, candidate, gates) return { "schema_version": "trtmc.qualification-result/v1", @@ -291,7 +290,9 @@ def _sts_samples(path: Path, count: int, prefix: str) -> list[dict[str, Any]]: raise QualificationError(f"STSBenchmark row {dataset_index} must be an object") sentence1 = row.get("sentence1") sentence2 = row.get("sentence2") - if not all(isinstance(value, str) and value.strip() for value in (sentence1, sentence2)): + if not all( + isinstance(value, str) and value.strip() for value in (sentence1, sentence2) + ): raise QualificationError( f"STSBenchmark row {dataset_index} must contain two sentences" ) @@ -302,9 +303,7 @@ def _sts_samples(path: Path, count: int, prefix: str) -> list[dict[str, Any]]: f"STSBenchmark row {dataset_index} has an invalid score" ) from error if not math.isfinite(score): - raise QualificationError( - f"STSBenchmark row {dataset_index} has a non-finite score" - ) + raise QualificationError(f"STSBenchmark row {dataset_index} has a non-finite score") pair_id = f"stsbenchmark-{dataset_index:06d}" for suffix, side, sentence in ( ("a", "sentence1", sentence1), @@ -387,9 +386,7 @@ def _compare_encoder_embeddings( "sentence2", }: raise QualificationError(f"encoder pair {pair_id!r} is incomplete") - reference_similarity = _vector_cosine( - expected["sentence1"][1], expected["sentence2"][1] - ) + reference_similarity = _vector_cosine(expected["sentence1"][1], expected["sentence2"][1]) candidate_similarity = _vector_cosine(actual["sentence1"], actual["sentence2"]) delta = abs(candidate_similarity - reference_similarity) score = float(expected["sentence1"][0]["score"]) @@ -412,9 +409,7 @@ def _compare_encoder_embeddings( pass_rate = passed_vectors / len(vector_cosines) max_pair_delta = max(pair_deltas) status = ( - "passed" - if pass_rate >= minimum_rate and max_pair_delta <= maximum_pair_delta - else "failed" + "passed" if pass_rate >= minimum_rate and max_pair_delta <= maximum_pair_delta else "failed" ) return { "status": status, @@ -471,16 +466,12 @@ def _finite_vector(values: Sequence[Any], label: str) -> list[float]: def _vector_cosine(left: Sequence[float], right: Sequence[float]) -> float: if len(left) != len(right) or not left: - raise QualificationError( - f"encoder vector dimensions differ: {len(left)} != {len(right)}" - ) + raise QualificationError(f"encoder vector dimensions differ: {len(left)} != {len(right)}") left_norm = math.sqrt(math.fsum(value * value for value in left)) right_norm = math.sqrt(math.fsum(value * value for value in right)) if left_norm <= 0.0 or right_norm <= 0.0: raise QualificationError("encoder vector must have a nonzero norm") - cosine = math.fsum(a * b for a, b in zip(left, right, strict=True)) / ( - left_norm * right_norm - ) + cosine = math.fsum(a * b for a, b in zip(left, right, strict=True)) / (left_norm * right_norm) return max(-1.0, min(1.0, cosine)) @@ -516,9 +507,9 @@ def _pearson(left: Sequence[float], right: Sequence[float]) -> float | None: ) if denominator <= 0.0: return None - return math.fsum( - a * b for a, b in zip(centered_left, centered_right, strict=True) - ) / denominator + return ( + math.fsum(a * b for a, b in zip(centered_left, centered_right, strict=True)) / denominator + ) def _etth1( @@ -762,8 +753,7 @@ def _image_classification_parity( reference_path = output / "reference.json" _json(request_path, reference_request) runner = ( - context.repository - / "tools/benchmark_qualification/references/timm_image_classification.py" + context.repository / "tools/benchmark_qualification/references/timm_image_classification.py" ) completed = run_command( [ @@ -793,13 +783,9 @@ def _image_classification_parity( } for sample in selected ] - actual, bundle = _candidate_outputs( - case, context, output, "classify", candidate_requests - ) + actual, bundle = _candidate_outputs(case, context, output, "classify", candidate_requests) rows = [] - for sample, reference_sample, candidate_sample in zip( - selected, expected, actual, strict=True - ): + for sample, reference_sample, candidate_sample in zip(selected, expected, actual, strict=True): reference_class = reference_sample.get("top_class") candidate_class = candidate_sample.get("top_class") if any( @@ -827,15 +813,12 @@ def _image_classification_parity( if not 0.0 <= minimum_agreement <= 1.0: raise QualificationError("min_top1_agreement must be in [0, 1]") if maximum_drop < 0.0 or not math.isfinite(maximum_drop): - raise QualificationError( - "max_top1_accuracy_drop_from_hf must be finite and nonnegative" - ) + raise QualificationError("max_top1_accuracy_drop_from_hf must be finite and nonnegative") agreement = sum(row["passed"] for row in rows) / len(rows) reference_accuracy = sum(row["reference_correct"] for row in rows) / len(rows) candidate_accuracy = sum(row["candidate_correct"] for row in rows) / len(rows) passed = ( - agreement >= minimum_agreement - and candidate_accuracy >= reference_accuracy - maximum_drop + agreement >= minimum_agreement and candidate_accuracy >= reference_accuracy - maximum_drop ) return { "schema_version": "trtmc.qualification-result/v1", @@ -952,17 +935,13 @@ def _speech_transcription_parity( "request": {**dict(candidate_request), "audio_path": audio_path}, } ) - actual, bundle = _candidate_outputs( - case, context, output, "transcribe", candidate_requests - ) + actual, bundle = _candidate_outputs(case, context, output, "transcribe", candidate_requests) rows = [] gold_reference_counts = [0, 0] gold_candidate_counts = [0, 0] parity_counts = [0, 0] - for source, reference_sample, candidate_sample in zip( - selected, expected, actual, strict=True - ): + for source, reference_sample, candidate_sample in zip(selected, expected, actual, strict=True): reference_text = reference_sample.get("text") candidate_text = candidate_sample.get("text") if not isinstance(reference_text, str) or not reference_text.strip(): @@ -1027,6 +1006,282 @@ def _speech_transcription_parity( } +def _image_feature_knn_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + bank_per_class = _positive_int( + configured.get("bank_samples_per_class"), "accuracy.bank_samples_per_class" + ) + query_per_class = _positive_int( + configured.get("query_samples_per_class"), "accuracy.query_samples_per_class" + ) + bank, queries = _image_feature_samples(dataset.path, bank_per_class, query_per_class) + samples = [*bank, *queries] + + reference = configured.get("reference") + if not isinstance(reference, Mapping): + raise QualificationError("image-feature Accuracy reference must be an object") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("image-feature Accuracy reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"image-feature reference runner is not family-owned: {runner}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "batch_size": int(reference.get("batch_size", 16)), + "samples": samples, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"image-feature Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(samples): + raise QualificationError("image-feature reference returned an invalid sample set") + + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"image_path": sample["image_path"]}, + } + for sample in samples + ] + actual, bundle = _candidate_outputs( + case, context, output, "extract_features", candidate_requests + ) + reference_vectors = [ + _finite_vector(sample.get("pooler_output", []), "HF image feature") for sample in expected + ] + candidate_vectors = [ + _finite_vector(sample.get("pooler_output", []), "TRTMC image feature") for sample in actual + ] + if any(len(left) != len(right) for left, right in zip(reference_vectors, candidate_vectors)): + raise QualificationError("image-feature reference and candidate dimensions differ") + + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("image-feature Accuracy gate must be an object") + minimum_cosine = float(gate.get("min_pooler_cosine", 0.999)) + minimum_rate = float(gate.get("min_vector_pass_rate", 1.0)) + minimum_agreement = float(gate.get("min_knn_top1_agreement", 0.98)) + maximum_drop = float(gate.get("max_knn_accuracy_drop_from_hf", 0.02)) + if not -1.0 <= minimum_cosine <= 1.0: + raise QualificationError("min_pooler_cosine must be in [-1, 1]") + if not 0.0 <= minimum_rate <= 1.0 or not 0.0 <= minimum_agreement <= 1.0: + raise QualificationError("image-feature pass rates must be in [0, 1]") + if maximum_drop < 0.0 or not math.isfinite(maximum_drop): + raise QualificationError("max_knn_accuracy_drop_from_hf must be finite and nonnegative") + + vector_rows = [] + cosines = [] + for sample, left, right in zip(samples, reference_vectors, candidate_vectors, strict=True): + cosine = _vector_cosine(left, right) + cosines.append(cosine) + vector_rows.append( + { + "sample_id": sample["sample_id"], + "split": sample["split"], + "label": sample["label"], + "pooler_cosine": cosine, + "passed": cosine >= minimum_cosine, + } + ) + + bank_count = len(bank) + knn_k = min(_positive_int(configured.get("knn_k", 10), "accuracy.knn_k"), bank_count) + temperature = float(configured.get("knn_temperature", 0.07)) + if not math.isfinite(temperature) or temperature <= 0.0: + raise QualificationError("accuracy.knn_temperature must be finite and positive") + labels = [sample["label"] for sample in bank] + reference_predictions = _knn_predictions( + reference_vectors[:bank_count], labels, reference_vectors[bank_count:], knn_k, temperature + ) + candidate_predictions = _knn_predictions( + candidate_vectors[:bank_count], labels, candidate_vectors[bank_count:], knn_k, temperature + ) + query_rows = [] + for sample, expected_label, candidate_label in zip( + queries, reference_predictions, candidate_predictions, strict=True + ): + query_rows.append( + { + "sample_id": sample["sample_id"], + "label": sample["label"], + "reference_top1": expected_label, + "candidate_top1": candidate_label, + "reference_correct": expected_label == sample["label"], + "candidate_correct": candidate_label == sample["label"], + "passed": candidate_label == expected_label, + } + ) + vector_pass_rate = sum(row["passed"] for row in vector_rows) / len(vector_rows) + agreement = sum(row["passed"] for row in query_rows) / len(query_rows) + reference_accuracy = sum(row["reference_correct"] for row in query_rows) / len(query_rows) + candidate_accuracy = sum(row["candidate_correct"] for row in query_rows) / len(query_rows) + accuracy_drop = reference_accuracy - candidate_accuracy + passed = ( + vector_pass_rate >= minimum_rate + and agreement >= minimum_agreement + and accuracy_drop <= maximum_drop + ) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "bank_samples": bank_count, + "query_samples": len(queries), + "vector_pass_rate": vector_pass_rate, + "mean_pooler_cosine": math.fsum(cosines) / len(cosines), + "min_pooler_cosine": min(cosines), + "knn_top1_agreement": agreement, + "reference_knn_top1_accuracy": reference_accuracy, + "candidate_knn_top1_accuracy": candidate_accuracy, + "knn_accuracy_drop_from_hf": accuracy_drop, + }, + "gate": { + "min_pooler_cosine": minimum_cosine, + "min_vector_pass_rate": minimum_rate, + "min_knn_top1_agreement": minimum_agreement, + "max_knn_accuracy_drop_from_hf": maximum_drop, + }, + "samples": vector_rows, + "queries": query_rows, + } + + +def _image_feature_samples( + dataset_path: Path, bank_per_class: int, query_per_class: int +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + payload = json.loads(dataset_path.read_text(encoding="utf-8")) + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or not requests: + raise QualificationError("image-feature dataset must contain requests") + root = dataset_path.parent.resolve() + bank_path: Path | None = None + query_paths: list[Path] = [] + for request in requests: + inputs = request.get("inputs") if isinstance(request, Mapping) else None + if not isinstance(inputs, Mapping): + raise QualificationError("image-feature dataset request must contain inputs") + configured_bank = _dataset_child(root, inputs.get("bank_manifest"), "bank manifest") + if bank_path is not None and bank_path != configured_bank: + raise QualificationError("image-feature dataset requests use different banks") + bank_path = configured_bank + query_paths.append(_dataset_child(root, inputs.get("query_manifest"), "query manifest")) + assert bank_path is not None + bank = _labeled_image_manifest(bank_path, root, "bank") + queries = [ + sample + for query_path in query_paths + for sample in _labeled_image_manifest(query_path, root, "query") + ] + return ( + _take_per_class(bank, bank_per_class, "bank"), + _take_per_class(queries, query_per_class, "query"), + ) + + +def _dataset_child(root: Path, value: Any, label: str) -> Path: + if not isinstance(value, str) or not value: + raise QualificationError(f"image-feature dataset omitted its {label}") + path = (root / value).resolve() + if root not in path.parents or not path.is_file(): + raise QualificationError(f"image-feature {label} is unavailable: {path}") + return path + + +def _labeled_image_manifest(path: Path, root: Path, split: str) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + samples = payload.get("samples") if isinstance(payload, Mapping) else None + if not isinstance(samples, list) or not samples: + raise QualificationError(f"image-feature {split} manifest has no samples") + result = [] + for index, sample in enumerate(samples): + relative = sample.get("image") if isinstance(sample, Mapping) else None + label = sample.get("label") if isinstance(sample, Mapping) else None + if not isinstance(relative, str) or isinstance(label, bool) or not isinstance(label, int): + raise QualificationError(f"image-feature {split} sample {index} is invalid") + image = (path.parent / relative).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError(f"image-feature {split} image is unavailable: {image}") + result.append( + { + "sample_id": f"{split}-{int(sample.get('source_index', index)):06d}", + "image_path": str(image), + "label": label, + "split": split, + } + ) + return result + + +def _take_per_class( + samples: Sequence[Mapping[str, Any]], count: int, split: str +) -> list[dict[str, Any]]: + classes = sorted({int(sample["label"]) for sample in samples}) + selected = [] + for label in classes: + matches = [dict(sample) for sample in samples if sample["label"] == label][:count] + if len(matches) != count: + raise QualificationError( + f"image-feature {split} class {label} has {len(matches)} samples; {count} required" + ) + selected.extend(matches) + return selected + + +def _knn_predictions( + bank: Sequence[Sequence[float]], + labels: Sequence[int], + queries: Sequence[Sequence[float]], + k: int, + temperature: float, +) -> list[int]: + predictions = [] + for query in queries: + neighbors = sorted( + ( + (_vector_cosine(query, vector), label) + for vector, label in zip(bank, labels, strict=True) + ), + reverse=True, + )[:k] + scores: dict[int, float] = {} + for similarity, label in neighbors: + scores[label] = scores.get(label, 0.0) + math.exp(similarity / temperature) + predictions.append(max(scores, key=lambda label: (scores[label], -label))) + return predictions + + def _speech_audio(request: Mapping[str, Any]) -> str: messages = request.get("messages") if not isinstance(messages, list): diff --git a/tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml b/tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml new file mode 100644 index 0000000000..bb5cbc4915 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: dinov3-beans-knn-v1 + version: AI-Lab-Makerere/beans@27aa014ce09b193e1a6f58112d4a66e0eddb69c5 + sha256: 77ea4c7f6204e212d92f4b01b8c47a8d1f2db59b7448f266c5b3a9ebc418b0f3 + source: + mode: manual + path: dinov3-beans-knn-v1/dataset.json + instructions: https://huggingface.co/datasets/AI-Lab-Makerere/beans + license: MIT +selection: + method: first-per-class +input: + adapter: image_path +metric: + name: image_feature_knn_parity diff --git a/tools/benchmark_qualification/benchmarks/image_features_performance.yaml b/tools/benchmark_qualification/benchmarks/image_features_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/image_features_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 96c720bf17..960d21a781 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -513,6 +513,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/test_impact.py", "tools/benchmark_qualification/__init__.py", "tools/benchmark_qualification/accuracy.py", + "tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml", "tools/benchmark_qualification/benchmarks/embedding_performance.yaml", "tools/benchmark_qualification/benchmarks/encoder_performance.yaml", "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", @@ -520,6 +521,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", + "tools/benchmark_qualification/benchmarks/image_features_performance.yaml", "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 5a2d30f394..7234e98f97 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -192,11 +192,7 @@ def reference(command, *_args, **_kwargs): captured.update(json.loads(request.read_text(encoding="utf-8"))) output.write_text( json.dumps( - { - "samples": [ - {"sample_id": "sample", "prompt": "Question", "token_ids": [1]} - ] - } + {"samples": [{"sample_id": "sample", "prompt": "Question", "token_ids": [1]}]} ), encoding="utf-8", ) @@ -235,9 +231,7 @@ def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) "local_files_only": True, "trust_remote_code": True, } - assert hf_text_generation._precision_load_options("fp16") == { - "torch_dtype": "fp16" - } + assert hf_text_generation._precision_load_options("fp16") == {"torch_dtype": "fp16"} def test_image_classification_accuracy_compares_top1_and_gold_accuracy( @@ -385,6 +379,149 @@ def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> N assert resolved == {"image_path": str(image.resolve()), "batch_size": 1} +def test_image_feature_accuracy_compares_vectors_and_knn_utility( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "beans" + (dataset_root / "images/train").mkdir(parents=True) + (dataset_root / "images/test").mkdir(parents=True) + (dataset_root / "queries").mkdir() + for split in ("train", "test"): + for index in range(2): + (dataset_root / f"images/{split}/{index}.jpg").write_bytes(b"fixture") + (dataset_root / "bank.json").write_text( + json.dumps( + { + "samples": [ + {"image": "images/train/0.jpg", "label": 0, "source_index": 0}, + {"image": "images/train/1.jpg", "label": 1, "source_index": 1}, + ] + } + ), + encoding="utf-8", + ) + (dataset_root / "queries/test.json").write_text( + json.dumps( + { + "samples": [ + {"image": "../images/test/0.jpg", "label": 0, "source_index": 0}, + {"image": "../images/test/1.jpg", "label": 1, "source_index": 1}, + ] + } + ), + encoding="utf-8", + ) + dataset_path = dataset_root / "dataset.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "inputs": { + "bank_manifest": "bank.json", + "query_manifest": "queries/test.json", + } + } + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/dinov3/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="dinov3", + name="beans-knn-parity", + benchmark="beans_image_feature_knn", + candidate={ + "family": "dinov3", + "checkpoint": "facebook/example", + "task": "image_features", + "precision": "fp16", + "build": {"max_sequence_length": 1}, + }, + values={ + "bank_samples_per_class": 1, + "query_samples_per_class": 1, + "knn_k": 1, + "knn_temperature": 0.07, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "min_pooler_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "min_knn_top1_agreement": 1.0, + "max_knn_accuracy_drop_from_hf": 0.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("dinov3-beans-knn-v1", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + vectors = [[1.0, 0.0], [0.0, 1.0], [0.9, 0.1], [0.1, 0.9]] + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + samples = json.loads(request.read_text(encoding="utf-8"))["samples"] + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": sample["sample_id"], "pooler_output": vector} + for sample, vector in zip(samples, vectors, strict=True) + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "image_feature_knn_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"pooler_output": vector} for vector in vectors], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["vector_pass_rate"] == 1.0 + assert result["metrics"]["knn_top1_agreement"] == 1.0 + assert result["metrics"]["candidate_knn_top1_accuracy"] == 1.0 + + def test_timm_classification_uses_a_generic_task_reference_adapter() -> None: assert "timm-classification" in task_reference.ADAPTERS assert task_reference.LOADERS["timm-classification"] is task_reference._load_vision @@ -395,9 +532,7 @@ def test_chat_asr_uses_the_generic_asr_reference_loader() -> None: assert task_reference.LOADERS["hf-chat-asr"] is task_reference._load_asr -def test_speech_accuracy_reports_reference_and_labeled_wer( - tmp_path: Path, monkeypatch -) -> None: +def test_speech_accuracy_reports_reference_and_labeled_wer(tmp_path: Path, monkeypatch) -> None: data_root = tmp_path / "datasets" audio_root = data_root / "speech/audio" audio_root.mkdir(parents=True) @@ -639,9 +774,7 @@ def candidate(_case, _context, _output, _operation, requests): assert result["status"] == "passed" assert captured_reference["task"] == "seq2seq-lm" assert captured_reference["output_token_policy"] == "strip-start-and-eos" - assert captured_reference["generation"]["source_language_placement"] == ( - "replace-final-unk" - ) + assert captured_reference["generation"]["source_language_placement"] == ("replace-final-unk") assert captured_reference["samples"] == [ {"sample_id": "translation-0", "prompt": "The house is wonderful."} ] @@ -1130,9 +1263,7 @@ def test_shared_performance_definitions_own_complete_reference_timing() -> None: "input_preparation_included", "asset_loading_included", } - assert timing_contract( - runner=str(case.values["reference"]["runner"]), declared=declared - ) + assert timing_contract(runner=str(case.values["reference"]["runner"]), declared=declared) def test_internal_automation_is_separate_from_the_installed_benchmark() -> None: @@ -1264,9 +1395,7 @@ def complete(command, *_args, **_kwargs): assert inherited.read_text(encoding="utf-8") == f"{parent_packages.resolve()}\n" -def test_bundle_preparation_uses_the_selected_runtime( - tmp_path: Path, monkeypatch -) -> None: +def test_bundle_preparation_uses_the_selected_runtime(tmp_path: Path, monkeypatch) -> None: case = QualificationCase( kind="accuracy", model="example-model", From dab17d29ed3a5a774156713a9c9f51e22c358894 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 18:42:24 +0000 Subject: [PATCH 033/100] feat(benchmark): qualify diffusion text generation Signed-off-by: chaofengw --- .../benchmark/nemotron-labs-diffusion-8b.yaml | 62 ++++++++++++ .../tests/benchmark/reference.py | 96 +++++++++++++++++++ tools/benchmark_qualification/accuracy.py | 16 +++- tools/tests/test_model_benchmark.py | 10 +- 4 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml create mode 100644 families/nemotron_labs_diffusion/tests/benchmark/reference.py diff --git a/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml b/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml new file mode 100644 index 0000000000..f77a8c23b7 --- /dev/null +++ b/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-labs-diffusion-8b +candidate: + family: nemotron_labs_diffusion + checkpoint: nvidia/Nemotron-Labs-Diffusion-8B + revision: 16c67f0560b912e93e0cabb6e0c4f5c3086d95fc + task: text_generation + precision: bf16 + trust_remote_code: true + bundle: nemotron-labs-diffusion-8b.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-ar-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + command: reference.py + precision: fp32 + request: &ar-generation + max_new_tokens: 32 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: true + enable_thinking: false + do_sample: false + generation_mode: ar + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-ar-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *ar-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: hf-eager + model_class: auto + generation_method: ar-generate + precision: fp32 + output_contract: exact-token-ids diff --git a/families/nemotron_labs_diffusion/tests/benchmark/reference.py b/families/nemotron_labs_diffusion/tests/benchmark/reference.py new file mode 100644 index 0000000000..a30a4d1d47 --- /dev/null +++ b/families/nemotron_labs_diffusion/tests/benchmark/reference.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Nemotron Labs Diffusion autoregressive reference.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Sequence + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def _inputs(tokenizer: Any, prompt: str, generation: dict[str, Any]) -> Any: + if not generation.get("use_chat_template", False): + return tokenizer(prompt, return_tensors="pt") + rendered = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=bool(generation.get("enable_thinking", False)), + ) + return tokenizer(rendered, return_tensors="pt", add_special_tokens=False) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + generation = request.get("generation") + if not isinstance(samples, list) or not samples or not isinstance(generation, dict): + raise ValueError("reference request must contain samples and generation") + if generation.get("generation_mode", "ar") != "ar": + raise ValueError("this qualification reference supports generation_mode=ar") + + import torch + from transformers import AutoModel, AutoTokenizer + + model_id = str(request["model"]) + revision = request.get("revision") + options = { + "trust_remote_code": bool(request.get("trust_remote_code", True)), + **({"revision": revision} if revision else {}), + } + tokenizer = AutoTokenizer.from_pretrained(model_id, **options) + model = AutoModel.from_pretrained( + model_id, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + **options, + ).eval().to("cuda") + max_new_tokens = int(generation.get("max_new_tokens", 20)) + results = [] + for sample in samples: + encoded = _inputs(tokenizer, str(sample["prompt"]), generation) + input_ids = encoded["input_ids"].to(model.device) + with torch.inference_mode(): + generated = model.ar_generate( + input_ids, + max_new_tokens=max_new_tokens, + temperature=float(generation.get("temperature", 0.0)), + eos_token_id=tokenizer.eos_token_id, + ) + if isinstance(generated, tuple): + generated = generated[0] + token_ids = generated[0, input_ids.shape[-1] :][:max_new_tokens].cpu().tolist() + results.append( + { + "sample_id": str(sample["sample_id"]), + "prompt": str(sample["prompt"]), + "token_ids": token_ids, + "text": tokenizer.decode(token_ids, skip_special_tokens=True), + } + ) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index dabdab79c7..207dd97677 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -113,7 +113,21 @@ def _text_generation_parity( request_path = output / "reference-request.json" reference_path = output / "reference.json" _json(request_path, reference_request) - runner = context.repository / "tools/benchmark_qualification/references/hf_text_generation.py" + runner_value = reference.get("command") + if runner_value is None: + runner = ( + context.repository + / "tools/benchmark_qualification/references/hf_text_generation.py" + ) + else: + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("text-generation reference.command must be a string") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError( + f"text-generation reference runner is not family-owned: {runner}" + ) completed = run_command( [ str(reference_python(case, context)), diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 7234e98f97..7f678dc91f 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -134,6 +134,10 @@ def test_accuracy_forwards_declared_reference_model_load_options( tmp_path: Path, monkeypatch, ) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") case = QualificationCase( kind="accuracy", model="example-model", @@ -154,13 +158,14 @@ def test_accuracy_forwards_declared_reference_model_load_options( "prompt_token_limit": 8, "truncation_side": "left", "reference": { + "command": "reference.py", "precision": "fp16", "experts_implementation": "batched_mm", }, "request": {"max_new_tokens": 1, "temperature": 0.0}, "gate": {"min_pass_rate": 1.0, "allowed_failures": 0}, }, - source=tmp_path / "example.yaml", + source=profile, reference_requirements=None, ) dataset_path = tmp_path / "mmlu.json" @@ -185,8 +190,10 @@ def test_accuracy_forwards_declared_reference_model_load_options( verbose=False, ) captured: dict[str, object] = {} + captured_command: list[str] = [] def reference(command, *_args, **_kwargs): + captured_command.extend(command) request = Path(command[command.index("--request") + 1]) output = Path(command[command.index("--output") + 1]) captured.update(json.loads(request.read_text(encoding="utf-8"))) @@ -215,6 +222,7 @@ def reference(command, *_args, **_kwargs): assert captured["revision"] == "a" * 40 assert captured["trust_remote_code"] is True assert captured["experts_implementation"] == "batched_mm" + assert runner in [Path(argument) for argument in captured_command] def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) -> None: From 84a1cddbc96ff227d21fed1ed1129d6541a91300 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 19:23:18 +0000 Subject: [PATCH 034/100] feat(benchmark): qualify detection and segmentation Add auto-discovered DETR and SegFormer Accuracy and Performance profiles. Compare DETR classes, boxes, and scores against Transformers and compare SegFormer masks by pixel accuracy and mean IoU on public Imagenette inputs. Expose model-agnostic detection parity and correct the existing vision reference summaries so both tasks run through the public trtmc-bench workflow. Signed-off-by: chaofengw --- .../performance/baselines/task_reference.py | 18 +- .../trtmc_benchmark/tests/test_perf_matrix.py | 34 ++ .../detr/tests/benchmark/detr-resnet-50.yaml | 54 +++ families/detr/tests/benchmark/reference.py | 88 +++++ .../segformer/tests/benchmark/reference.py | 82 ++++ .../tests/benchmark/segformer-b0-ade.yaml | 46 +++ tools/benchmark_qualification/accuracy.py | 351 +++++++++++++++++- .../imagenette_detection_parity.yaml | 20 + .../imagenette_segmentation_parity.yaml | 20 + .../object_detection_performance.yaml | 15 + .../semantic_segmentation_performance.yaml | 15 + tools/perf_matrix.py | 91 +++++ tools/tests/test_architecture.py | 4 + tools/tests/test_model_benchmark.py | 214 +++++++++++ 14 files changed, 1047 insertions(+), 5 deletions(-) create mode 100644 families/detr/tests/benchmark/detr-resnet-50.yaml create mode 100644 families/detr/tests/benchmark/reference.py create mode 100644 families/segformer/tests/benchmark/reference.py create mode 100644 families/segformer/tests/benchmark/segformer-b0-ade.yaml create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/object_detection_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index a51b6080a9..7549ea1084 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -1622,6 +1622,12 @@ def invoke() -> Mapping[str, Any]: "detections": int(results["scores"].shape[0]), "image_height": height, "image_width": width, + "boxes": results["boxes"].float().cpu().reshape(-1).tolist(), + "scores": results["scores"].float().cpu().tolist(), + "class_ids": results["labels"].cpu().tolist(), + "shape": [int(results["scores"].shape[0]), 4], + "coordinates": "xyxy", + "units": "pixels", } elif arguments.family == "dinov3": @@ -1660,9 +1666,15 @@ def invoke() -> Mapping[str, Any]: def invoke() -> Mapping[str, Any]: with torch.inference_mode(): - logits = model(**inputs).logits - masks = logits.argmax(dim=1) - return {"mask_count": int(masks.shape[0]), **_tensor_summary(masks)} + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(height, width)] + )[0] + return { + "num_masks": 1, + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + } elif arguments.family == "sam3": processor = _load_sam3_processor(transformers, arguments.model, processor_kwargs) diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index b323e3492d..44fd31af85 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -3178,3 +3178,37 @@ def test_output_contracts_are_closed_and_semantic(tmp_path: Path) -> None: ) with pytest.raises(perf.PerfMatrixError, match="unsupported output contract"): perf._contract_name(bad) + + +def test_detection_output_contract_matches_by_class_and_iou() -> None: + entry = SimpleNamespace( + spec={ + "baseline": { + "output_contract": "detection-parity", + "min_box_iou": 0.9, + "max_score_abs_error": 0.05, + } + } + ) + candidate = { + "output_summary": { + "boxes": [10.0, 10.0, 50.0, 50.0], + "scores": [0.91], + "class_ids": [7], + } + } + reference = { + "output_summary": { + "boxes": [[10.5, 10.0, 50.0, 50.0]], + "scores": [0.9], + "class_ids": [7], + } + } + + matched, reason, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert reason == "" + assert evidence["minimum_box_iou"] >= 0.9 + reference["output_summary"]["class_ids"] = [8] + assert perf._output_contract(entry, candidate, reference)[0] is False diff --git a/families/detr/tests/benchmark/detr-resnet-50.yaml b/families/detr/tests/benchmark/detr-resnet-50.yaml new file mode 100644 index 0000000000..471203b346 --- /dev/null +++ b/families/detr/tests/benchmark/detr-resnet-50.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: detr-resnet-50 +candidate: + family: detr + checkpoint: facebook/detr-resnet-50 + revision: 1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b + task: object_detection + precision: fp16 + bundle: detr-resnet-50.bundle + build: + max_sequence_length: 1 + image_height: 796 + image_width: 1333 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.5 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.5 + max_score_abs_error: 0.05 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-vision + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.5 + max_score_abs_error: 0.05 diff --git a/families/detr/tests/benchmark/reference.py b/families/detr/tests/benchmark/reference.py new file mode 100644 index 0000000000..aca5be52de --- /dev/null +++ b/families/detr/tests/benchmark/reference.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers DETR reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Sequence + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForObjectDetection + + model_id = str(request["model"]) + revision = request.get("revision") + options = {"revision": revision} if revision else {} + processor = AutoImageProcessor.from_pretrained(model_id, **options) + model = ( + AutoModelForObjectDetection.from_pretrained( + model_id, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + **options, + ) + .eval() + .to("cuda") + ) + score_threshold = float(request.get("request", {}).get("score_threshold", 0.5)) + result = [] + for sample in samples: + image = Image.open(str(sample["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + with torch.inference_mode(): + outputs = model(**inputs) + detections = processor.post_process_object_detection( + outputs, + threshold=score_threshold, + target_sizes=torch.tensor([[image.height, image.width]], device=model.device), + )[0] + boxes = detections["boxes"].float().cpu().tolist() + result.append( + { + "sample_id": str(sample["sample_id"]), + "image_height": image.height, + "image_width": image.width, + "boxes": boxes, + "scores": detections["scores"].float().cpu().tolist(), + "class_ids": detections["labels"].cpu().tolist(), + } + ) + arguments.output.write_text(json.dumps({"samples": result}, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/segformer/tests/benchmark/reference.py b/families/segformer/tests/benchmark/reference.py new file mode 100644 index 0000000000..efa3734065 --- /dev/null +++ b/families/segformer/tests/benchmark/reference.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers SegFormer reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Sequence + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForSemanticSegmentation + + model_id = str(request["model"]) + revision = request.get("revision") + options = {"revision": revision} if revision else {} + processor = AutoImageProcessor.from_pretrained(model_id, use_fast=False, **options) + model = ( + AutoModelForSemanticSegmentation.from_pretrained( + model_id, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + **options, + ) + .eval() + .to("cuda") + ) + result = [] + for sample in samples: + image = Image.open(str(sample["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + with torch.inference_mode(): + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(image.height, image.width)] + )[0].cpu() + result.append( + { + "sample_id": str(sample["sample_id"]), + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + "mask": mask.reshape(-1).tolist(), + } + ) + arguments.output.write_text(json.dumps({"samples": result}, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/segformer/tests/benchmark/segformer-b0-ade.yaml b/families/segformer/tests/benchmark/segformer-b0-ade.yaml new file mode 100644 index 0000000000..37c095207c --- /dev/null +++ b/families/segformer/tests/benchmark/segformer-b0-ade.yaml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: segformer-b0-ade +candidate: + family: segformer + checkpoint: nvidia/segformer-b0-finetuned-ade-512-512 + revision: 489d5cd81a0b59fab9b7ea758d3548ebe99677da + task: segmentation + precision: fp16 + bundle: segformer-b0-ade.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-segmentation-parity + benchmark: imagenette_segmentation_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + gate: + min_pixel_accuracy: 0.99 + min_mean_iou: 0.94 + min_sample_pass_rate: 0.9 + +performance: + - name: segment + benchmark: semantic_segmentation_performance + operation: segment + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-vision + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: segmentation-shape diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 207dd97677..1160ef8002 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -45,6 +45,10 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _speech_transcription_parity(case, context, dataset, output) elif metric_name == "image_feature_knn_parity": result = _image_feature_knn_parity(case, context, dataset, output) + elif metric_name == "object_detection_parity": + result = _object_detection_parity(case, context, dataset, output) + elif metric_name == "semantic_segmentation_parity": + result = _semantic_segmentation_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -116,8 +120,7 @@ def _text_generation_parity( runner_value = reference.get("command") if runner_value is None: runner = ( - context.repository - / "tools/benchmark_qualification/references/hf_text_generation.py" + context.repository / "tools/benchmark_qualification/references/hf_text_generation.py" ) else: if not isinstance(runner_value, str) or not runner_value: @@ -711,6 +714,350 @@ def _candidate_outputs( return outputs, bundle +def _image_parity_samples(dataset: Dataset, sample_limit: int, task: str) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError(f"{task} dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"{task} dataset requires at least {sample_limit} requests") + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + relative = request.get("image") if isinstance(request, Mapping) else None + if not isinstance(relative, str) or not relative: + raise QualificationError(f"{task} request {index} has no image") + image = (root / relative).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError(f"{task} request {index} image is unavailable: {image}") + selected.append( + { + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + } + ) + return selected + + +def _family_image_reference( + case: QualificationCase, + context: RuntimeContext, + output: Path, + selected: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + reference = case.values.get("reference") + if not isinstance(reference, Mapping): + raise QualificationError("image Accuracy reference must be an object") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("image Accuracy reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"image reference runner is not family-owned: {runner}") + request = case.values.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("image Accuracy request must be an object") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "request": dict(request), + "samples": list(selected), + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"image Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("image Accuracy reference returned an invalid sample set") + return expected + + +def _box_iou(left: Sequence[float], right: Sequence[float]) -> float: + intersection_width = max(0.0, min(left[2], right[2]) - max(left[0], right[0])) + intersection_height = max(0.0, min(left[3], right[3]) - max(left[1], right[1])) + intersection = intersection_width * intersection_height + left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1]) + right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1]) + union = left_area + right_area - intersection + return intersection / union if union > 0.0 else 0.0 + + +def _detections(value: Mapping[str, Any], label: str) -> list[dict[str, Any]]: + boxes = value.get("boxes") + scores = value.get("scores") + classes = value.get("class_ids", value.get("classes")) + if not isinstance(boxes, list) or not isinstance(scores, list) or not isinstance(classes, list): + raise QualificationError(f"{label} detection output is incomplete") + if boxes and isinstance(boxes[0], list): + rows = boxes + else: + if len(boxes) % 4: + raise QualificationError(f"{label} detection boxes are not xyxy rows") + rows = [boxes[index : index + 4] for index in range(0, len(boxes), 4)] + if len(rows) != len(scores) or len(rows) != len(classes): + raise QualificationError(f"{label} detection arrays have different lengths") + detections = [] + for index, (box, score, class_id) in enumerate(zip(rows, scores, classes, strict=True)): + if ( + not isinstance(box, list) + or len(box) != 4 + or isinstance(class_id, bool) + or not isinstance(class_id, int) + ): + raise QualificationError(f"{label} detection {index} is invalid") + numeric_box = [float(coordinate) for coordinate in box] + numeric_score = float(score) + if not all(math.isfinite(value) for value in (*numeric_box, numeric_score)): + raise QualificationError(f"{label} detection {index} is not finite") + detections.append({"box": numeric_box, "score": numeric_score, "class_id": class_id}) + return detections + + +def _match_detections( + candidate: Sequence[Mapping[str, Any]], reference: Sequence[Mapping[str, Any]] +) -> list[dict[str, Any]]: + matched_reference: set[int] = set() + matches = [] + for detection in sorted(candidate, key=lambda item: -float(item["score"])): + choices = [ + (_box_iou(detection["box"], expected["box"]), index) + for index, expected in enumerate(reference) + if index not in matched_reference + and int(detection["class_id"]) == int(expected["class_id"]) + ] + if not choices: + continue + iou, index = max(choices) + matched_reference.add(index) + matches.append( + { + "class_id": int(detection["class_id"]), + "iou": iou, + "score_abs_error": abs( + float(detection["score"]) - float(reference[index]["score"]) + ), + } + ) + return matches + + +def _object_detection_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "object-detection") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("object-detection request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "detect", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("object-detection gate must be an object") + minimum_iou = float(gate.get("min_box_iou", 0.5)) + maximum_score_error = float(gate.get("max_score_abs_error", 0.05)) + minimum_match_rate = float(gate.get("min_detection_match_rate", 1.0)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 for value in (minimum_iou, minimum_match_rate, minimum_sample_rate) + ): + raise QualificationError("object-detection rates and IoU must be in [0, 1]") + if maximum_score_error < 0.0 or not math.isfinite(maximum_score_error): + raise QualificationError("max_score_abs_error must be finite and nonnegative") + + rows = [] + total_matches = total_detections = 0 + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_detections = _detections(candidate_sample, "candidate") + reference_detections = _detections(reference_sample, "reference") + matches = _match_detections(candidate_detections, reference_detections) + denominator = max(len(candidate_detections), len(reference_detections), 1) + match_rate = len(matches) / denominator + minimum_sample_iou = min((match["iou"] for match in matches), default=1.0) + maximum_sample_score_error = max( + (match["score_abs_error"] for match in matches), default=0.0 + ) + passed = ( + match_rate >= minimum_match_rate + and minimum_sample_iou >= minimum_iou + and maximum_sample_score_error <= maximum_score_error + ) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": passed, + "candidate_detections": len(candidate_detections), + "reference_detections": len(reference_detections), + "matched_detections": len(matches), + "detection_match_rate": match_rate, + "min_box_iou": minimum_sample_iou, + "max_score_abs_error": maximum_sample_score_error, + } + ) + total_matches += len(matches) + total_detections += denominator + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + detection_match_rate = total_matches / total_detections + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "detection_match_rate": detection_match_rate, + "min_box_iou": min(row["min_box_iou"] for row in rows), + "max_score_abs_error": max(row["max_score_abs_error"] for row in rows), + }, + "gate": { + "min_box_iou": minimum_iou, + "max_score_abs_error": maximum_score_error, + "min_detection_match_rate": minimum_match_rate, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _semantic_mask(value: Mapping[str, Any], label: str) -> tuple[int, int, list[int]]: + height = value.get("height") + width = value.get("width") + mask = value.get("mask") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or not isinstance(mask, list) + or len(mask) != height * width + ): + raise QualificationError(f"{label} semantic-segmentation output is invalid") + if any(isinstance(item, bool) or not isinstance(item, int) for item in mask): + raise QualificationError(f"{label} semantic-segmentation mask is not integral") + return height, width, mask + + +def _semantic_segmentation_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "semantic-segmentation") + expected = _family_image_reference(case, context, output, selected) + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "segment", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("semantic-segmentation gate must be an object") + minimum_pixel_accuracy = float(gate.get("min_pixel_accuracy", 0.99)) + minimum_mean_iou = float(gate.get("min_mean_iou", 0.94)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 + for value in (minimum_pixel_accuracy, minimum_mean_iou, minimum_sample_rate) + ): + raise QualificationError("semantic-segmentation gates must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + left_height, left_width, left = _semantic_mask(candidate_sample, "candidate") + right_height, right_width, right = _semantic_mask(reference_sample, "reference") + if (left_height, left_width) != (right_height, right_width): + pixel_accuracy = mean_iou = 0.0 + else: + pixel_accuracy = sum(a == b for a, b in zip(left, right, strict=True)) / len(left) + class_ious = [] + for class_id in sorted(set(left) | set(right)): + intersection = sum(a == class_id and b == class_id for a, b in zip(left, right)) + union = sum(a == class_id or b == class_id for a, b in zip(left, right)) + if union: + class_ious.append(intersection / union) + mean_iou = math.fsum(class_ious) / len(class_ious) if class_ious else 0.0 + rows.append( + { + "sample_id": sample["sample_id"], + "passed": pixel_accuracy >= minimum_pixel_accuracy and mean_iou >= minimum_mean_iou, + "pixel_accuracy": pixel_accuracy, + "mean_iou": mean_iou, + "candidate_shape": [left_height, left_width], + "reference_shape": [right_height, right_width], + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "mean_pixel_accuracy": math.fsum(row["pixel_accuracy"] for row in rows) / len(rows), + "mean_iou": math.fsum(row["mean_iou"] for row in rows) / len(rows), + "min_pixel_accuracy": min(row["pixel_accuracy"] for row in rows), + "min_mean_iou": min(row["mean_iou"] for row in rows), + }, + "gate": { + "min_pixel_accuracy": minimum_pixel_accuracy, + "min_mean_iou": minimum_mean_iou, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _image_classification_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml new file mode 100644 index 0000000000..61c379ee10 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: object_detection_parity diff --git a/tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml new file mode 100644 index 0000000000..1c418f5a1f --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: semantic_segmentation_parity diff --git a/tools/benchmark_qualification/benchmarks/object_detection_performance.yaml b/tools/benchmark_qualification/benchmarks/object_detection_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/object_detection_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml b/tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 19fb62f4e6..a854df6eed 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -65,6 +65,7 @@ OUTPUT_CONTRACTS = { "audio-shape", "classification-top-class", + "detection-parity", "disparity-parity", "embedding-shape", "exact-text", @@ -1364,6 +1365,8 @@ def _output_contract( right_shape = tuple(right.get(name) for name in ("num_masks", "height", "width")) matched = None not in left_shape and left_shape == right_shape return matched, "segmentation output shape differs" if not matched else "", None + if contract == "detection-parity": + return _detection_parity(entry, left, right) if contract == "classification-top-class": left_class = left.get("top_class") right_class = right.get("top_class") @@ -1991,6 +1994,94 @@ def _localization_contract( return distance <= limit, "localization text distance exceeds the contract", evidence +def _detection_values(summary: Mapping[str, Any]) -> list[dict[str, Any]] | None: + boxes = summary.get("boxes") + scores = summary.get("scores") + classes = summary.get("class_ids", summary.get("classes")) + if not isinstance(boxes, list) or not isinstance(scores, list) or not isinstance(classes, list): + return None + if boxes and isinstance(boxes[0], list): + rows = boxes + else: + if len(boxes) % 4: + return None + rows = [boxes[index : index + 4] for index in range(0, len(boxes), 4)] + if len(rows) != len(scores) or len(rows) != len(classes): + return None + detections = [] + for box, score, class_id in zip(rows, scores, classes, strict=True): + if ( + not isinstance(box, list) + or len(box) != 4 + or isinstance(class_id, bool) + or not isinstance(class_id, int) + ): + return None + try: + coordinates = tuple(float(value) for value in box) + confidence = float(score) + except (TypeError, ValueError): + return None + if not all(math.isfinite(value) for value in (*coordinates, confidence)): + return None + detections.append( + {"box": coordinates, "score": confidence, "class_id": class_id} + ) + return detections + + +def _detection_parity( + entry: ResolvedEntry, + left: Mapping[str, Any], + right: Mapping[str, Any], +) -> tuple[bool, str, dict[str, Any] | None]: + candidate = _detection_values(left) + reference = _detection_values(right) + if candidate is None or reference is None: + return False, "detection output is invalid", None + if len(candidate) != len(reference): + return ( + False, + "detection count differs", + {"candidate_count": len(candidate), "reference_count": len(reference)}, + ) + matched_reference: set[int] = set() + matches = [] + for detection in sorted(candidate, key=lambda item: -float(item["score"])): + choices = [ + (_box_iou(detection["box"], expected["box"]), index) + for index, expected in enumerate(reference) + if index not in matched_reference + and int(detection["class_id"]) == int(expected["class_id"]) + ] + if not choices: + return False, "detection classes differ", None + iou, index = max(choices) + matched_reference.add(index) + matches.append( + ( + iou, + abs(float(detection["score"]) - float(reference[index]["score"])), + ) + ) + minimum_iou = min((match[0] for match in matches), default=1.0) + maximum_score_error = max((match[1] for match in matches), default=0.0) + required_iou = float(entry.spec["baseline"].get("min_box_iou", 0.5)) + allowed_score_error = float(entry.spec["baseline"].get("max_score_abs_error", 0.05)) + evidence = { + "detections": len(matches), + "minimum_box_iou": minimum_iou, + "required_box_iou": required_iou, + "maximum_score_abs_error": maximum_score_error, + "allowed_score_abs_error": allowed_score_error, + } + if minimum_iou < required_iou: + return False, "detection box IoU is below the contract", evidence + if maximum_score_error > allowed_score_error: + return False, "detection score error exceeds the contract", evidence + return True, "", evidence + + def _box_iou(left: tuple[float, ...], right: tuple[float, ...]) -> float: lx1, ly1, lx2, ly2 = left rx1, ry1, rx2, ry2 = right diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 960d21a781..7fa4f0af37 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -523,13 +523,17 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", "tools/benchmark_qualification/benchmarks/image_features_performance.yaml", "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", + "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", + "tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", "tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 7f678dc91f..3fc379d414 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -387,6 +387,220 @@ def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> N assert resolved == {"image_path": str(image.resolve()), "batch_size": 1} +def test_object_detection_accuracy_matches_classes_boxes_and_scores( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "Imagenette" + dataset_root.mkdir() + image = dataset_root / "image.jpeg" + image.write_bytes(b"fixture") + dataset_path = dataset_root / "manifest.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), + encoding="utf-8", + ) + profile = tmp_path / "families/detr/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="detr", + name="detection-parity", + benchmark="imagenette_detection_parity", + candidate={ + "family": "detr", + "checkpoint": "facebook/example", + "task": "object_detection", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "request": {"score_threshold": 0.5}, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "min_box_iou": 0.9, + "max_score_abs_error": 0.05, + "min_detection_match_rate": 1.0, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("imagenette-validation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + + def reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "image", + "boxes": [[10.0, 20.0, 50.0, 70.0]], + "scores": [0.9], + "class_ids": [3], + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "object_detection_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [ + { + "boxes": [10.5, 20.0, 50.0, 70.0], + "scores": [0.89], + "class_ids": [3], + } + ], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["detection_match_rate"] == 1.0 + assert result["metrics"]["min_box_iou"] >= 0.9 + + +def test_semantic_segmentation_accuracy_compares_pixel_and_class_iou( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "Imagenette" + dataset_root.mkdir() + image = dataset_root / "image.jpeg" + image.write_bytes(b"fixture") + dataset_path = dataset_root / "manifest.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), + encoding="utf-8", + ) + profile = tmp_path / "families/segformer/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="segformer", + name="segmentation-parity", + benchmark="imagenette_segmentation_parity", + candidate={ + "family": "segformer", + "checkpoint": "nvidia/example", + "task": "segmentation", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "min_pixel_accuracy": 0.75, + "min_mean_iou": 0.58, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("imagenette-validation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + + def reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "image", + "height": 2, + "width": 2, + "mask": [0, 0, 1, 1], + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "semantic_segmentation_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"height": 2, "width": 2, "mask": [0, 0, 1, 0]}], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["mean_pixel_accuracy"] == 0.75 + assert result["metrics"]["mean_iou"] >= 0.58 + + def test_image_feature_accuracy_compares_vectors_and_knn_utility( tmp_path: Path, monkeypatch ) -> None: From c6c85e23c3add5881ea0d9ed0150a5aa2cc244dc Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 19:39:20 +0000 Subject: [PATCH 035/100] feat(benchmark): qualify YOLO detection Signed-off-by: chaofengw --- families/yolo11/tests/benchmark/reference.py | 282 +++++++++++++++++++ families/yolo11/tests/benchmark/yolo11n.yaml | 52 ++++ families/yolov8/tests/benchmark/reference.py | 282 +++++++++++++++++++ families/yolov8/tests/benchmark/yolov8n.yaml | 52 ++++ 4 files changed, 668 insertions(+) create mode 100644 families/yolo11/tests/benchmark/reference.py create mode 100644 families/yolo11/tests/benchmark/yolo11n.yaml create mode 100644 families/yolov8/tests/benchmark/reference.py create mode 100644 families/yolov8/tests/benchmark/yolov8n.yaml diff --git a/families/yolo11/tests/benchmark/reference.py b/families/yolo11/tests/benchmark/reference.py new file mode 100644 index 0000000000..63e5b440df --- /dev/null +++ b/families/yolo11/tests/benchmark/reference.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLO11 archive for qualification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +ARCHIVE = "yolo11n.pt" + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +def _load_model(model_dir: Path, precision: str) -> Any: + import torch + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + archive = model_dir / ARCHIVE + if not archive.is_file(): + raise FileNotFoundError(f"YOLO11 model directory has no {ARCHIVE}: {model_dir}") + blob = torch.load(str(archive), map_location="cpu", weights_only=False) + model = blob.get("model") if isinstance(blob, dict) else None + if model is None: + raise ValueError(f"YOLO11 archive has no model entry: {archive}") + return model.eval().to(device="cuda", dtype=dtype) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Any, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + from ultralytics.utils.ops import non_max_suppression + + with torch.inference_mode(): + raw = model(pixels) + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + kept = non_max_suppression( + raw, + conf_thres=threshold, + iou_thres=0.7, + max_det=300, + )[0] + rows = kept.detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in rows + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(rows), + "boxes": boxes, + "scores": [float(row[4]) for row in rows], + "class_ids": [int(row[5]) for row in rows], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), next(model.parameters()).dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolo11" or arguments.operation != "detect": + raise ValueError("YOLO11 reference only supports yolo11 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLO11 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), next(model.parameters()).dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolo11/tests/benchmark/yolo11n.yaml b/families/yolo11/tests/benchmark/yolo11n.yaml new file mode 100644 index 0000000000..be9e997b15 --- /dev/null +++ b/families/yolo11/tests/benchmark/yolo11n.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolo11n +candidate: + family: yolo11 + checkpoint: Ultralytics/YOLO11 + revision: 8b8ac7d1fae7468f85dbf89670dd66f41f485aab + task: object_detection + precision: fp16 + bundle: yolo11n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.9 + max_score_abs_error: 0.005 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov8/tests/benchmark/reference.py b/families/yolov8/tests/benchmark/reference.py new file mode 100644 index 0000000000..1a37c75797 --- /dev/null +++ b/families/yolov8/tests/benchmark/reference.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLOv8 archive for qualification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +ARCHIVE = "yolov8n.pt" + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +def _load_model(model_dir: Path, precision: str) -> Any: + import torch + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + archive = model_dir / ARCHIVE + if not archive.is_file(): + raise FileNotFoundError(f"YOLOv8 model directory has no {ARCHIVE}: {model_dir}") + blob = torch.load(str(archive), map_location="cpu", weights_only=False) + model = blob.get("model") if isinstance(blob, dict) else None + if model is None: + raise ValueError(f"YOLOv8 archive has no model entry: {archive}") + return model.eval().to(device="cuda", dtype=dtype) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Any, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + from ultralytics.utils.ops import non_max_suppression + + with torch.inference_mode(): + raw = model(pixels) + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + kept = non_max_suppression( + raw, + conf_thres=threshold, + iou_thres=0.7, + max_det=300, + )[0] + rows = kept.detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in rows + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(rows), + "boxes": boxes, + "scores": [float(row[4]) for row in rows], + "class_ids": [int(row[5]) for row in rows], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), next(model.parameters()).dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolov8" or arguments.operation != "detect": + raise ValueError("YOLOv8 reference only supports yolov8 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLOv8 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), next(model.parameters()).dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolov8/tests/benchmark/yolov8n.yaml b/families/yolov8/tests/benchmark/yolov8n.yaml new file mode 100644 index 0000000000..eb6478fbf4 --- /dev/null +++ b/families/yolov8/tests/benchmark/yolov8n.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov8n +candidate: + family: yolov8 + checkpoint: Ultralytics/YOLOv8 + revision: 27f858f2dc238503a66ce10248cf7ec71a3ded5d + task: object_detection + precision: fp16 + bundle: yolov8n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.9 + max_score_abs_error: 0.005 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 From 5a542629862352f984361a1bf33d4f069585850c Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 19:49:21 +0000 Subject: [PATCH 036/100] feat(benchmark): qualify YOLOv10 detection Signed-off-by: chaofengw --- families/yolov10/tests/benchmark/reference.py | 283 ++++++++++++++++++ .../yolov10/tests/benchmark/yolov10n.yaml | 52 ++++ .../yolov10/tests/benchmark/yolov10s.yaml | 52 ++++ .../yolov10/tests/benchmark/yolov10x.yaml | 52 ++++ 4 files changed, 439 insertions(+) create mode 100644 families/yolov10/tests/benchmark/reference.py create mode 100644 families/yolov10/tests/benchmark/yolov10n.yaml create mode 100644 families/yolov10/tests/benchmark/yolov10s.yaml create mode 100644 families/yolov10/tests/benchmark/yolov10x.yaml diff --git a/families/yolov10/tests/benchmark/reference.py b/families/yolov10/tests/benchmark/reference.py new file mode 100644 index 0000000000..5a92d44d6b --- /dev/null +++ b/families/yolov10/tests/benchmark/reference.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLOv10 archive for qualification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +def _load_model(model_dir: Path, precision: str) -> Any: + import torch + from ultralytics import YOLO + + from families.yolov10.checkpoint import Checkpoint + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + architecture = config.get("model") if isinstance(config, dict) else None + supported = {"yolov10n.yaml", "yolov10s.yaml", "yolov10x.yaml"} + if not isinstance(architecture, str) or architecture not in supported: + raise ValueError(f"unsupported YOLOv10 reference architecture: {architecture!r}") + model = YOLO(architecture, task="detect").model + checkpoint = Checkpoint.open(model_dir, framework="pt") + raw = {name: reader.get_tensor(name) for name, reader in checkpoint.tensor_map.items()} + state = { + name[len("model.") :]: value for name, value in raw.items() if name.startswith("model.") + } + missing, _ = model.load_state_dict(state, strict=False) + if missing: + raise ValueError(f"YOLOv10 reference is missing checkpoint tensors: {missing[:5]}") + return model.eval().to(device="cuda", dtype=dtype) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Any, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + raw = model(pixels) + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + rows = raw[0] + kept = rows[rows[:, 4] >= threshold].detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in kept + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(kept), + "boxes": boxes, + "scores": [float(row[4]) for row in kept], + "class_ids": [int(row[5]) for row in kept], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), next(model.parameters()).dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolov10" or arguments.operation != "detect": + raise ValueError("YOLOv10 reference only supports yolov10 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLOv10 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), next(model.parameters()).dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolov10/tests/benchmark/yolov10n.yaml b/families/yolov10/tests/benchmark/yolov10n.yaml new file mode 100644 index 0000000000..4b631bd899 --- /dev/null +++ b/families/yolov10/tests/benchmark/yolov10n.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov10n +candidate: + family: yolov10 + checkpoint: jameslahm/yolov10n + revision: 3b91f578c5b492080f191a26aa35711bb44f608c + task: object_detection + precision: fp16 + bundle: yolov10n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.9 + max_score_abs_error: 0.005 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov10/tests/benchmark/yolov10s.yaml b/families/yolov10/tests/benchmark/yolov10s.yaml new file mode 100644 index 0000000000..141c6d9bab --- /dev/null +++ b/families/yolov10/tests/benchmark/yolov10s.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov10s +candidate: + family: yolov10 + checkpoint: jameslahm/yolov10s + revision: 9096a248f4ab5dd21b3d9e9fa0295534db1b87ff + task: object_detection + precision: fp16 + bundle: yolov10s.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.9 + max_score_abs_error: 0.005 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov10/tests/benchmark/yolov10x.yaml b/families/yolov10/tests/benchmark/yolov10x.yaml new file mode 100644 index 0000000000..1f7a4c9995 --- /dev/null +++ b/families/yolov10/tests/benchmark/yolov10x.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov10x +candidate: + family: yolov10 + checkpoint: jameslahm/yolov10x + revision: 6d7c9b6871ffa50dfd2764fa896bc0e33294233c + task: object_detection + precision: fp16 + bundle: yolov10x.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.9 + max_score_abs_error: 0.005 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 From cf9a3850628d28b63575237943caf9d64bc0593d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 20:01:45 +0000 Subject: [PATCH 037/100] feat(benchmark): qualify YOLOv5 detection Signed-off-by: chaofengw --- families/yolov5/tests/benchmark/reference.py | 387 +++++++++++++++++++ families/yolov5/tests/benchmark/yolov5n.yaml | 52 +++ 2 files changed, 439 insertions(+) create mode 100644 families/yolov5/tests/benchmark/reference.py create mode 100644 families/yolov5/tests/benchmark/yolov5n.yaml diff --git a/families/yolov5/tests/benchmark/reference.py b/families/yolov5/tests/benchmark/reference.py new file mode 100644 index 0000000000..0456198aa1 --- /dev/null +++ b/families/yolov5/tests/benchmark/reference.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLOv5 archive for qualification.""" + +from __future__ import annotations + +import argparse +import copy +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + stages: Any + anchors: Any + strides: Any + head: tuple[Any, ...] + biases: tuple[Any, ...] + per_anchor: int + dtype: Any + + +def _load_model(model_dir: Path, precision: str) -> Reference: + import torch + from ultralytics.nn.tasks import parse_model + + from families.yolov5.checkpoint import _collect, _install_placeholders + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + archive_path = model_dir / "yolov5n.pt" + if not archive_path.is_file(): + raise FileNotFoundError(f"YOLOv5 model directory has no yolov5n.pt: {model_dir}") + _install_placeholders() + blob = torch.load(str(archive_path), map_location="cpu", weights_only=False) + archive = blob.get("model") if isinstance(blob, dict) else None + if archive is None: + raise ValueError(f"YOLOv5 archive has no model entry: {archive_path}") + spec = copy.deepcopy(archive.__dict__["yaml"]) + spec["head"].pop() + stages, _ = parse_model(spec, ch=3, verbose=False) + weights: dict[str, Any] = {} + _collect(archive, "", weights) + body = { + name[len("model.") :]: value.float() + for name, value in weights.items() + if name.startswith("model.") + } + head_index = str(len(stages)) + missing, unexpected = stages.load_state_dict( + {name: value for name, value in body.items() if not name.startswith(f"{head_index}.")}, + strict=False, + ) + if missing: + raise ValueError(f"YOLOv5 reference is missing checkpoint tensors: {missing[:5]}") + if any(not name.endswith("num_batches_tracked") for name in unexpected): + raise ValueError(f"YOLOv5 reference has unexpected checkpoint tensors: {unexpected[:5]}") + for module in stages.modules(): + if isinstance(module, torch.nn.BatchNorm2d): + module.eps = 1e-3 + classes = len(archive.__dict__["names"]) + strides = archive.__dict__["stride"].float().to(device="cuda", dtype=dtype) + anchors = weights[f"model.{head_index}.anchors"].float().to(device="cuda", dtype=dtype) + head = tuple( + body[f"{head_index}.m.{level}.weight"].to(device="cuda", dtype=dtype) + for level in range(len(strides)) + ) + biases = tuple( + body[f"{head_index}.m.{level}.bias"].to(device="cuda", dtype=dtype) + for level in range(len(strides)) + ) + return Reference( + stages=stages.eval().to(device="cuda", dtype=dtype), + anchors=anchors, + strides=strides, + head=head, + biases=biases, + per_anchor=classes + 5, + dtype=dtype, + ) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Reference, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + import torchvision + + with torch.inference_mode(): + outputs: list[Any] = [] + tensor = pixels + for stage in model.stages: + if stage.f != -1: + taken = [tensor if index == -1 else outputs[index] for index in stage.f] + tensor = stage(taken) + else: + tensor = stage(tensor) + outputs.append(tensor) + + levels = [] + for level, source_index in enumerate((17, 20, 23)): + raw = torch.nn.functional.conv2d( + outputs[source_index], + model.head[level], + model.biases[level], + ) + rows, columns = raw.shape[-2], raw.shape[-1] + values = raw.view(1, len(model.anchors[level]), model.per_anchor, rows, columns) + values = values.permute(0, 1, 3, 4, 2).sigmoid() + grid_y, grid_x = torch.meshgrid( + torch.arange(rows, device=raw.device), + torch.arange(columns, device=raw.device), + indexing="ij", + ) + grid = torch.stack((grid_x, grid_y), dim=-1).to(dtype=model.dtype) + centre = (values[..., 0:2] * 2 - 0.5 + grid) * model.strides[level] + extent = ( + (values[..., 2:4] * 2) ** 2 + * model.anchors[level].view(1, -1, 1, 1, 2) + * model.strides[level] + ) + levels.append( + torch.cat((centre, extent, values[..., 4:]), dim=-1).reshape( + 1, -1, model.per_anchor + ) + ) + predictions = torch.cat(levels, dim=1)[0] + confidence = predictions[:, 4:5] * predictions[:, 5:] + best, labels = confidence.max(dim=1) + survivors = best > threshold + centre = predictions[survivors, 0:2] + extent = predictions[survivors, 2:4] + corners = torch.cat((centre - extent / 2, centre + extent / 2), dim=1) + order = torchvision.ops.batched_nms( + corners, + best[survivors], + labels[survivors], + 0.7, + )[:300] + kept = torch.cat( + ( + corners[order], + best[survivors][order, None], + labels[survivors][order, None].to(dtype=model.dtype), + ), + dim=1, + ) + values = kept.detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in values + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(values), + "boxes": boxes, + "scores": [float(row[4]) for row in values], + "class_ids": [int(row[5]) for row in values], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), model.dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolov5" or arguments.operation != "detect": + raise ValueError("YOLOv5 reference only supports yolov5 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLOv5 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), model.dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolov5/tests/benchmark/yolov5n.yaml b/families/yolov5/tests/benchmark/yolov5n.yaml new file mode 100644 index 0000000000..31a201cccf --- /dev/null +++ b/families/yolov5/tests/benchmark/yolov5n.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov5n +candidate: + family: yolov5 + checkpoint: Ultralytics/YOLOv5 + revision: 5bca797074771ecdfd6267d6e9be32ee201d937b + task: object_detection + precision: fp16 + bundle: yolov5n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-detection-parity + benchmark: imagenette_detection_parity + samples: 10 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + min_box_iou: 0.9 + max_score_abs_error: 0.005 + min_detection_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 From 1ac5bc0f9edde1b36ccf823df77ff055eb488975 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 20:16:44 +0000 Subject: [PATCH 038/100] feat(benchmark): qualify SAM segmentation Signed-off-by: chaofengw --- .../trtmc_benchmark/tests/test_perf_matrix.py | 32 +++ families/sam/tests/benchmark/reference.py | 268 ++++++++++++++++++ .../sam/tests/benchmark/sam-vit-base.yaml | 51 ++++ tools/benchmark_qualification/accuracy.py | 148 ++++++++++ ...agenette_prompted_segmentation_parity.yaml | 20 ++ tools/perf_matrix.py | 86 ++++++ tools/tests/test_architecture.py | 1 + tools/tests/test_model_benchmark.py | 22 ++ 8 files changed, 628 insertions(+) create mode 100644 families/sam/tests/benchmark/reference.py create mode 100644 families/sam/tests/benchmark/sam-vit-base.yaml create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 44fd31af85..112ab00b98 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -3212,3 +3212,35 @@ def test_detection_output_contract_matches_by_class_and_iou() -> None: assert evidence["minimum_box_iou"] >= 0.9 reference["output_summary"]["class_ids"] = [8] assert perf._output_contract(entry, candidate, reference)[0] is False + + +def test_prompted_mask_contract_matches_binary_semantics() -> None: + entry = SimpleNamespace( + spec={"baseline": {"output_contract": "prompted-mask-parity", "min_mask_iou": 0.7}} + ) + candidate = { + "output_summary": { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "logits", + "masks": [2.0, -1.0, -1.0, 3.0], + } + } + reference = { + "output_summary": { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [1, 0, 0, 1], + } + } + + matched, reason, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert reason == "" + assert evidence == {"masks": 1, "minimum_mask_iou": 1.0, "required_mask_iou": 0.7} + reference["output_summary"]["masks"] = [0, 1, 1, 0] + assert perf._output_contract(entry, candidate, reference)[0] is False diff --git a/families/sam/tests/benchmark/reference.py b/families/sam/tests/benchmark/reference.py new file mode 100644 index 0000000000..a59f585c96 --- /dev/null +++ b/families/sam/tests/benchmark/reference.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers SAM reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import SamModel, SamProcessor + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = SamProcessor.from_pretrained(model_dir, use_fast=False) + model = SamModel.from_pretrained(model_dir, torch_dtype=dtype).eval().to("cuda") + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, request: Mapping[str, Any]) -> dict[str, Any]: + import torch + from PIL import Image + + image = Image.open(image_path).convert("RGB") + x = int(float(request.get("point_x", 0.5)) * image.width) + y = int(float(request.get("point_y", 0.5)) * image.height) + label = 1 if bool(request.get("is_foreground", True)) else 0 + encoded = reference.processor( + image, + input_points=[[[x, y]]], + input_labels=[[label]], + return_tensors="pt", + ) + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if isinstance(value, torch.Tensor) + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any]) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + outputs = reference.model(**inputs) + masks = reference.processor.image_processor.post_process_masks( + outputs.pred_masks.cpu(), + inputs["original_sizes"].cpu(), + inputs["reshaped_input_sizes"].cpu(), + binarize=False, + )[0] + masks = masks.detach().float() + height, width = (int(value) for value in masks.shape[-2:]) + count = int(masks.numel() // (height * width)) + scores = outputs.iou_scores.detach().float().cpu().reshape(-1).tolist() + return { + "segmented_images": 1, + "generated_masks": count, + "num_masks": count, + "mask_pixels": int(masks.numel()), + "height": height, + "width": width, + "mask_kind": "logits", + "masks": masks.reshape(-1).tolist(), + "iou_scores": scores, + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), controls) + results.append({"sample_id": str(sample["sample_id"]), **_infer(reference, inputs)}) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "sam" or arguments.operation != "segment_prompted": + raise ValueError("SAM reference only supports sam segment_prompted") + if arguments.mode != "pytorch-eager": + raise ValueError("SAM reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), request) + for _ in range(arguments.warmup): + _infer(reference, inputs) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sam/tests/benchmark/sam-vit-base.yaml b/families/sam/tests/benchmark/sam-vit-base.yaml new file mode 100644 index 0000000000..c876d980d4 --- /dev/null +++ b/families/sam/tests/benchmark/sam-vit-base.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: sam-vit-base +candidate: + family: sam + checkpoint: facebook/sam-vit-base + revision: 70c1a07f894ebb5b307fd9eaaee97b9dfc16068f + task: prompted_segmentation + precision: fp16 + bundle: sam-vit-base.bundle + build: {} + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-prompted-mask-parity + benchmark: imagenette_prompted_segmentation_parity + samples: 10 + request: &prompted-request + point_x: 0.5 + point_y: 0.5 + is_foreground: true + reference: + command: reference.py + precision: fp32 + gate: + min_mask_iou: 0.7 + min_mask_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: segment-prompted + benchmark: semantic_segmentation_performance + operation: segment_prompted + request: + <<: *prompted-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: prompted-mask-parity + min_mask_iou: 0.7 diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 1160ef8002..5d28225509 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -47,6 +47,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _image_feature_knn_parity(case, context, dataset, output) elif metric_name == "object_detection_parity": result = _object_detection_parity(case, context, dataset, output) + elif metric_name == "prompted_segmentation_parity": + result = _prompted_segmentation_parity(case, context, dataset, output) elif metric_name == "semantic_segmentation_parity": result = _semantic_segmentation_parity(case, context, dataset, output) else: @@ -975,6 +977,152 @@ def _semantic_mask(value: Mapping[str, Any], label: str) -> tuple[int, int, list return height, width, mask +def _binary_masks(value: Mapping[str, Any], label: str) -> tuple[int, int, list[list[bool]]]: + height = value.get("height") + width = value.get("width") + count = value.get("num_masks") + masks = value.get("masks") + kind = value.get("mask_kind", "logits") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or isinstance(count, bool) + or not isinstance(count, int) + or count < 1 + or not isinstance(masks, list) + or len(masks) != count * height * width + or kind not in {"logits", "probability", "binary"} + ): + raise QualificationError(f"{label} prompted-segmentation output is invalid") + try: + numeric = [float(value) for value in masks] + except (TypeError, ValueError) as error: + raise QualificationError( + f"{label} prompted-segmentation masks are not numeric" + ) from error + if not all(math.isfinite(value) for value in numeric): + raise QualificationError(f"{label} prompted-segmentation masks are not finite") + threshold = 0.0 if kind == "logits" else 0.5 + area = height * width + return height, width, [ + [value > threshold for value in numeric[index : index + area]] + for index in range(0, len(numeric), area) + ] + + +def _mask_iou(left: Sequence[bool], right: Sequence[bool]) -> float: + intersection = sum(a and b for a, b in zip(left, right, strict=True)) + union = sum(a or b for a, b in zip(left, right, strict=True)) + return intersection / union if union else 1.0 + + +def _match_masks(candidate: Sequence[Sequence[bool]], reference: Sequence[Sequence[bool]]) -> list[float]: + matched_reference: set[int] = set() + matches = [] + for mask in candidate: + choices = [ + (_mask_iou(mask, expected), index) + for index, expected in enumerate(reference) + if index not in matched_reference + ] + if not choices: + continue + iou, index = max(choices) + matched_reference.add(index) + matches.append(iou) + return matches + + +def _prompted_segmentation_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "prompted-segmentation") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("prompted-segmentation request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs( + case, context, output, "segment_prompted", candidate_requests + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("prompted-segmentation gate must be an object") + minimum_iou = float(gate.get("min_mask_iou", 0.7)) + minimum_match_rate = float(gate.get("min_mask_match_rate", 1.0)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 for value in (minimum_iou, minimum_match_rate, minimum_sample_rate) + ): + raise QualificationError("prompted-segmentation gates must be in [0, 1]") + + rows = [] + total_matches = total_masks = 0 + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + left_height, left_width, left = _binary_masks(candidate_sample, "candidate") + right_height, right_width, right = _binary_masks(reference_sample, "reference") + matches = ( + _match_masks(left, right) + if (left_height, left_width) == (right_height, right_width) + else [] + ) + denominator = max(len(left), len(right), 1) + match_rate = len(matches) / denominator + minimum_sample_iou = min(matches, default=0.0) + passed = match_rate >= minimum_match_rate and minimum_sample_iou >= minimum_iou + rows.append( + { + "sample_id": sample["sample_id"], + "passed": passed, + "candidate_masks": len(left), + "reference_masks": len(right), + "mask_match_rate": match_rate, + "min_mask_iou": minimum_sample_iou, + } + ) + total_matches += len(matches) + total_masks += denominator + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "mask_match_rate": total_matches / total_masks, + "min_mask_iou": min(row["min_mask_iou"] for row in rows), + }, + "gate": { + "min_mask_iou": minimum_iou, + "min_mask_match_rate": minimum_match_rate, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _semantic_segmentation_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml new file mode 100644 index 0000000000..ccc49d093a --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: prompted_segmentation_parity diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index a854df6eed..338d9d425e 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -85,6 +85,7 @@ "ocr-text", "offline-speech-shape", "pose-refinement-shape", + "prompted-mask-parity", "reranking-order", "robot-action-shape", "segmentation-shape", @@ -1365,6 +1366,8 @@ def _output_contract( right_shape = tuple(right.get(name) for name in ("num_masks", "height", "width")) matched = None not in left_shape and left_shape == right_shape return matched, "segmentation output shape differs" if not matched else "", None + if contract == "prompted-mask-parity": + return _prompted_mask_parity(entry, left, right) if contract == "detection-parity": return _detection_parity(entry, left, right) if contract == "classification-top-class": @@ -2030,6 +2033,89 @@ def _detection_values(summary: Mapping[str, Any]) -> list[dict[str, Any]] | None return detections +def _prompted_mask_values( + summary: Mapping[str, Any], +) -> tuple[int, int, list[list[bool]]] | None: + height = summary.get("height") + width = summary.get("width") + count = summary.get("num_masks") + masks = summary.get("masks") + kind = summary.get("mask_kind", "logits") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or isinstance(count, bool) + or not isinstance(count, int) + or count < 1 + or not isinstance(masks, list) + or len(masks) != count * height * width + or kind not in {"logits", "probability", "binary"} + ): + return None + if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in masks): + return None + numeric = [float(value) for value in masks] + if not all(math.isfinite(value) for value in numeric): + return None + threshold = 0.0 if kind == "logits" else 0.5 + area = height * width + return height, width, [ + [value > threshold for value in numeric[index : index + area]] + for index in range(0, len(numeric), area) + ] + + +def _binary_mask_iou(left: Sequence[bool], right: Sequence[bool]) -> float: + intersection = sum(a and b for a, b in zip(left, right, strict=True)) + union = sum(a or b for a, b in zip(left, right, strict=True)) + return intersection / union if union else 1.0 + + +def _prompted_mask_parity( + entry: ResolvedEntry, + left: Mapping[str, Any], + right: Mapping[str, Any], +) -> tuple[bool, str, dict[str, Any] | None]: + candidate = _prompted_mask_values(left) + reference = _prompted_mask_values(right) + if candidate is None or reference is None: + return False, "prompted mask output is invalid", None + if candidate[:2] != reference[:2] or len(candidate[2]) != len(reference[2]): + return ( + False, + "prompted mask shape or count differs", + { + "candidate_shape": [len(candidate[2]), *candidate[:2]], + "reference_shape": [len(reference[2]), *reference[:2]], + }, + ) + matched_reference: set[int] = set() + matches = [] + for mask in candidate[2]: + choices = [ + (_binary_mask_iou(mask, expected), index) + for index, expected in enumerate(reference[2]) + if index not in matched_reference + ] + iou, index = max(choices) + matched_reference.add(index) + matches.append(iou) + minimum_iou = min(matches) + required_iou = float(entry.spec["baseline"].get("min_mask_iou", 0.7)) + evidence = { + "masks": len(matches), + "minimum_mask_iou": minimum_iou, + "required_mask_iou": required_iou, + } + if minimum_iou < required_iou: + return False, "prompted mask IoU is below the contract", evidence + return True, "", evidence + + def _detection_parity( entry: ResolvedEntry, left: Mapping[str, Any], diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 7fa4f0af37..06ceb792e3 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -524,6 +524,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/image_features_performance.yaml", "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 3fc379d414..9f8b7c9590 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -498,6 +498,28 @@ def reference(command, *_args, **_kwargs): assert result["metrics"]["min_box_iou"] >= 0.9 +def test_prompted_segmentation_masks_match_independent_of_order() -> None: + first = { + "num_masks": 2, + "height": 2, + "width": 2, + "mask_kind": "logits", + "masks": [1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0], + } + second = { + "num_masks": 2, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [0, 1, 1, 0, 1, 0, 0, 1], + } + + _, _, candidate = qualification_accuracy._binary_masks(first, "candidate") + _, _, reference = qualification_accuracy._binary_masks(second, "reference") + + assert qualification_accuracy._match_masks(candidate, reference) == [1.0, 1.0] + + def test_semantic_segmentation_accuracy_compares_pixel_and_class_iou( tmp_path: Path, monkeypatch ) -> None: From d1159aa06433fa4b19550fd082bb04dab20c8492 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 20:36:49 +0000 Subject: [PATCH 039/100] feat(benchmark): qualify Qwen vision language Signed-off-by: chaofengw --- .../trtmc_benchmark/tests/test_perf_matrix.py | 20 ++ .../qwen_vl/tests/benchmark/qwen25vl-3b.yaml | 53 ++++ .../qwen_vl/tests/benchmark/qwen3-vl-2b.yaml | 53 ++++ families/qwen_vl/tests/benchmark/reference.py | 289 ++++++++++++++++++ tools/benchmark_qualification/accuracy.py | 93 ++++++ .../imagenette_vision_language_parity.yaml | 20 ++ .../vision_language_performance.yaml | 15 + tools/perf_matrix.py | 14 + tools/tests/test_architecture.py | 2 + tools/tests/test_model_benchmark.py | 8 + 10 files changed, 567 insertions(+) create mode 100644 families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml create mode 100644 families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml create mode 100644 families/qwen_vl/tests/benchmark/reference.py create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/vision_language_performance.yaml diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 112ab00b98..0fb1df2b13 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -3244,3 +3244,23 @@ def test_prompted_mask_contract_matches_binary_semantics() -> None: assert evidence == {"masks": 1, "minimum_mask_iou": 1.0, "required_mask_iou": 0.7} reference["output_summary"]["masks"] = [0, 1, 1, 0] assert perf._output_contract(entry, candidate, reference)[0] is False + + +def test_vision_language_text_contract_allows_small_normalized_differences() -> None: + entry = SimpleNamespace( + spec={ + "baseline": { + "output_contract": "vision-language-text", + "max_normalized_edit_distance": 0.15, + } + } + ) + candidate = {"output_summary": {"text": "Red."}} + reference = {"output_summary": {"text": "red"}} + + matched, _, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert evidence["normalized_edit_distance"] <= 0.15 + candidate["output_summary"]["text"] = "blue" + assert perf._output_contract(entry, candidate, reference)[0] is False diff --git a/families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml b/families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml new file mode 100644 index 0000000000..b5c0c52694 --- /dev/null +++ b/families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen25vl-3b +candidate: + family: qwen_vl + checkpoint: Qwen/Qwen2.5-VL-3B-Instruct + revision: 66285546d2b821cf421d4f5eb2576359d3770cd3 + task: vision_language_generation + precision: fp32 + bundle: qwen25vl-3b-vl.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml b/families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml new file mode 100644 index 0000000000..eb637c6190 --- /dev/null +++ b/families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-vl-2b +candidate: + family: qwen_vl + checkpoint: Qwen/Qwen3-VL-2B-Instruct + revision: 89644892e4d85e24eaac8bacfd4f463576704203 + task: vision_language_generation + precision: bf16 + bundle: qwen3-vl-2b.bundle + build: + max_sequence_length: 256 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: bf16 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: bf16 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/qwen_vl/tests/benchmark/reference.py b/families/qwen_vl/tests/benchmark/reference.py new file mode 100644 index 0000000000..b6a7ba3c72 --- /dev/null +++ b/families/qwen_vl/tests/benchmark/reference.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers Qwen-VL reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import AutoModelForImageTextToText, AutoProcessor + + try: + dtype = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True) + model = ( + AutoModelForImageTextToText.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=dtype, + ) + .eval() + .to("cuda") + ) + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + rendered = reference.processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image", "image": image_path}, + {"type": "text", "text": prompt}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + if not isinstance(rendered, str) or not rendered.strip(): + raise ValueError("Qwen-VL chat template produced an empty prompt") + if not any(marker in rendered for marker in ("<|image_pad|>", "<|vision_start|>", "")): + raise ValueError("Qwen-VL chat template lost the image placeholder") + encoded = reference.processor(text=rendered, images=image, padding=True, return_tensors="pt") + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if hasattr(value, "to") + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any], max_new_tokens: int) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + generated = reference.model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = generated[0] + tokens = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in tokens.detach().cpu().tolist()] + return { + "text": reference.processor.decode(tokens, skip_special_tokens=True), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + prompt = str(controls.get("prompt", "")) + max_new_tokens = int(controls.get("max_new_tokens", 10)) + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append( + { + "sample_id": str(sample["sample_id"]), + **_infer(reference, inputs, max_new_tokens), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "qwen_vl" or arguments.operation != "generate": + raise ValueError("Qwen-VL reference only supports qwen_vl generate") + if arguments.mode != "pytorch-eager": + raise ValueError("Qwen-VL reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request.get("prompt", ""))) + max_new_tokens = int(request.get("max_new_tokens", 10)) + for _ in range(arguments.warmup): + _infer(reference, inputs, max_new_tokens) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs, max_new_tokens) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 5d28225509..a109832fb9 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -51,6 +51,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _prompted_segmentation_parity(case, context, dataset, output) elif metric_name == "semantic_segmentation_parity": result = _semantic_segmentation_parity(case, context, dataset, output) + elif metric_name == "vision_language_text_parity": + result = _vision_language_text_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -1123,6 +1125,97 @@ def _prompted_segmentation_parity( } +def _normalized_answer(value: Any) -> str: + return " ".join(str(value or "").casefold().split()).strip(".,!?;:'\"") + + +def _normalized_edit_distance(left: str, right: str) -> float: + if left == right: + return 0.0 + if not left or not right: + return 1.0 + previous = list(range(len(right) + 1)) + for row, left_character in enumerate(left, start=1): + current = [row] + for column, right_character in enumerate(right, start=1): + current.append( + min( + current[column - 1] + 1, + previous[column] + 1, + previous[column - 1] + (left_character != right_character), + ) + ) + previous = current + return previous[-1] / max(len(left), len(right)) + + +def _vision_language_text_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "vision-language") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("vision-language request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("vision-language gate must be an object") + maximum_distance = float(gate.get("max_normalized_edit_distance", 0.15)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= maximum_distance <= 1.0 or not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("vision-language gates must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_text = _normalized_answer(candidate_sample.get("text")) + reference_text = _normalized_answer(reference_sample.get("text")) + distance = _normalized_edit_distance(candidate_text, reference_text) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": bool(candidate_text) and distance <= maximum_distance, + "candidate_text": candidate_text, + "reference_text": reference_text, + "normalized_edit_distance": distance, + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "max_normalized_edit_distance": max(row["normalized_edit_distance"] for row in rows), + }, + "gate": { + "max_normalized_edit_distance": maximum_distance, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _semantic_segmentation_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml new file mode 100644 index 0000000000..944f02a17c --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: vision_language_text_parity diff --git a/tools/benchmark_qualification/benchmarks/vision_language_performance.yaml b/tools/benchmark_qualification/benchmarks/vision_language_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/vision_language_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 338d9d425e..44f9900dee 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -90,6 +90,7 @@ "robot-action-shape", "segmentation-shape", "transcription-text", + "vision-language-text", } REFERENCE_INPUTS = { "pytorch-lerobot-act": (("source_root", "lerobot_repo"),), @@ -1324,6 +1325,19 @@ def _output_contract( "maximum": limit, }, ) + if contract == "vision-language-text": + left_text = _normalized_text(left.get("text")).strip(".,!?;:'\"") + right_text = _normalized_text(right.get("text")).strip(".,!?;:'\"") + distance = _text_distance(left_text, right_text) + limit = float(entry.spec["baseline"].get("max_normalized_edit_distance", 0.15)) + return ( + bool(left_text) and distance <= limit, + "vision-language text distance exceeds the contract", + { + "normalized_edit_distance": distance, + "maximum": limit, + }, + ) if contract == "localization": return _localization_contract(entry, left, right) if contract in {"metric-geometry-shape", "molecular-structure-shape", "pose-refinement-shape"}: diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 06ceb792e3..5042d7bd32 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -526,6 +526,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", @@ -534,6 +535,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", + "tools/benchmark_qualification/benchmarks/vision_language_performance.yaml", "tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 9f8b7c9590..3b2eef0bcd 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -520,6 +520,14 @@ def test_prompted_segmentation_masks_match_independent_of_order() -> None: assert qualification_accuracy._match_masks(candidate, reference) == [1.0, 1.0] +def test_vision_language_text_distance_is_normalized() -> None: + left = qualification_accuracy._normalized_answer(" Red\ncar ") + right = qualification_accuracy._normalized_answer("red car") + + assert qualification_accuracy._normalized_edit_distance(left, right) == 0.0 + assert qualification_accuracy._normalized_edit_distance(left, "blue car") > 0.15 + + def test_semantic_segmentation_accuracy_compares_pixel_and_class_iou( tmp_path: Path, monkeypatch ) -> None: From c49e769a3d93317fabcb525439d8d08273d10e3a Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 20:47:06 +0000 Subject: [PATCH 040/100] feat(benchmark): qualify InternVL generation Signed-off-by: chaofengw --- .../tests/benchmark/internvl3-2b.yaml | 53 ++++ .../tests/benchmark/internvl3-8b.yaml | 53 ++++ .../internvl/tests/benchmark/reference.py | 292 ++++++++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 families/internvl/tests/benchmark/internvl3-2b.yaml create mode 100644 families/internvl/tests/benchmark/internvl3-8b.yaml create mode 100644 families/internvl/tests/benchmark/reference.py diff --git a/families/internvl/tests/benchmark/internvl3-2b.yaml b/families/internvl/tests/benchmark/internvl3-2b.yaml new file mode 100644 index 0000000000..256c1656f8 --- /dev/null +++ b/families/internvl/tests/benchmark/internvl3-2b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: internvl3-2b +candidate: + family: internvl + checkpoint: OpenGVLab/InternVL3-2B-hf + revision: cb57a075cb75a2e6d1b668b128d48bb00ae321d2 + task: vision_language_generation + precision: fp16 + bundle: internvl3-2b-vl.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/internvl/tests/benchmark/internvl3-8b.yaml b/families/internvl/tests/benchmark/internvl3-8b.yaml new file mode 100644 index 0000000000..bb7bcaf2e3 --- /dev/null +++ b/families/internvl/tests/benchmark/internvl3-8b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: internvl3-8b +candidate: + family: internvl + checkpoint: OpenGVLab/InternVL3-8B-hf + revision: 259a3b64a14623c0ec91a045cb43f7c5af5fa6af + task: vision_language_generation + precision: fp16 + bundle: internvl3-8b-vl.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/internvl/tests/benchmark/reference.py b/families/internvl/tests/benchmark/reference.py new file mode 100644 index 0000000000..7194f1a412 --- /dev/null +++ b/families/internvl/tests/benchmark/reference.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers InternVL reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import AutoModelForImageTextToText, AutoProcessor + + try: + dtype = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True) + model = ( + AutoModelForImageTextToText.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=dtype, + ) + .eval() + .to("cuda") + ) + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + rendered = reference.processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": prompt}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + if ( + not isinstance(rendered, str) + or not rendered.strip() + or prompt not in rendered + or "" not in rendered + ): + raise ValueError("InternVL chat template lost the image or text prompt") + encoded = reference.processor(text=rendered, images=image, padding=True, return_tensors="pt") + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if hasattr(value, "to") + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any], max_new_tokens: int) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + generated = reference.model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = generated[0] + tokens = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in tokens.detach().cpu().tolist()] + return { + "text": reference.processor.decode(tokens, skip_special_tokens=True), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + prompt = str(controls.get("prompt", "")) + max_new_tokens = int(controls.get("max_new_tokens", 10)) + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append( + { + "sample_id": str(sample["sample_id"]), + **_infer(reference, inputs, max_new_tokens), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "internvl" or arguments.operation != "generate": + raise ValueError("InternVL reference only supports internvl generate") + if arguments.mode != "pytorch-eager": + raise ValueError("InternVL reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request.get("prompt", ""))) + max_new_tokens = int(request.get("max_new_tokens", 10)) + for _ in range(arguments.warmup): + _infer(reference, inputs, max_new_tokens) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs, max_new_tokens) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) From 3cb78f8f5611a0837ec9002d573ecc2a9d063a5a Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 20:55:20 +0000 Subject: [PATCH 041/100] feat(benchmark): qualify Phi-4 multimodal Signed-off-by: chaofengw --- .../tests/benchmark/phi4-multimodal.yaml | 53 ++++ .../tests/benchmark/reference.py | 291 ++++++++++++++++++ 2 files changed, 344 insertions(+) create mode 100644 families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml create mode 100644 families/phi4_multimodal/tests/benchmark/reference.py diff --git a/families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml b/families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml new file mode 100644 index 0000000000..d808863a1e --- /dev/null +++ b/families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: phi4-multimodal +candidate: + family: phi4_multimodal + checkpoint: microsoft/Phi-4-multimodal-instruct + revision: 93f923e1a7727d1c4f446756212d9d3e8fcc5d81 + task: vision_language_generation + precision: fp16 + bundle: phi4-multimodal.bundle + build: + max_sequence_length: 768 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/phi4_multimodal/tests/benchmark/reference.py b/families/phi4_multimodal/tests/benchmark/reference.py new file mode 100644 index 0000000000..fa4d47d0c0 --- /dev/null +++ b/families/phi4_multimodal/tests/benchmark/reference.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers Phi-4 Multimodal reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor + + try: + dtype = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True) + model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True) + model_config._attn_implementation = "eager" + model_config._attn_implementation_internal = "eager" + model = ( + AutoModelForCausalLM.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=dtype, + config=model_config, + attn_implementation="eager", + ) + .eval() + .to("cuda") + ) + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + rendered = reference.processor.tokenizer.apply_chat_template( + [{"role": "user", "content": f"<|image_1|>{prompt}"}], + tokenize=False, + add_generation_prompt=True, + ) + if ( + not isinstance(rendered, str) + or not rendered.strip() + or not any( + marker in rendered + for marker in ("<|image_1|>", "<|image_pad|>", "<|vision_start|>", "") + ) + ): + raise ValueError("Phi-4 Multimodal chat template lost the image placeholder") + encoded = reference.processor(text=rendered, images=image, return_tensors="pt") + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if hasattr(value, "to") + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any], max_new_tokens: int) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + generated = reference.model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = generated[0] + tokens = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in tokens.detach().cpu().tolist()] + return { + "text": reference.processor.decode(tokens, skip_special_tokens=True), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + prompt = str(controls.get("prompt", "")) + max_new_tokens = int(controls.get("max_new_tokens", 10)) + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append( + { + "sample_id": str(sample["sample_id"]), + **_infer(reference, inputs, max_new_tokens), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "phi4_multimodal" or arguments.operation != "generate": + raise ValueError("Phi-4 Multimodal reference only supports phi4_multimodal generate") + if arguments.mode != "pytorch-eager": + raise ValueError("Phi-4 Multimodal reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request.get("prompt", ""))) + max_new_tokens = int(request.get("max_new_tokens", 10)) + for _ in range(arguments.warmup): + _infer(reference, inputs, max_new_tokens) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs, max_new_tokens) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) From 1936b70a28406ef68280dcb8678bf68e2d0445a2 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 21:05:35 +0000 Subject: [PATCH 042/100] feat(benchmark): qualify Nemotron VL embedding Signed-off-by: chaofengw --- .../benchmark/nemotron-embed-vl-1b-v2.yaml | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml diff --git a/families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml b/families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml new file mode 100644 index 0000000000..ed61663485 --- /dev/null +++ b/families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-embed-vl-1b-v2 +candidate: + family: eagle_vlm + checkpoint: nvidia/llama-nemotron-embed-vl-1b-v2 + revision: 582e3bf72aee355e3c59ed89de53543c5b0657ee + task: embedding + precision: fp16 + trust_remote_code: true + bundle: nemotron-embed-vl-1b-v2.bundle + build: + max_sequence_length: 512 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.995 + min_vector_pass_rate: 0.98 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: embed + benchmark: embedding_performance + operation: embed + request: + prompt: What is artificial intelligence? + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-embedding + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + reference_backend: hf_transformers + precision: fp32 + output_contract: embedding-shape From d7cd3eb9a4fae0e277bad8d28c8dff76bf140523 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 21:16:29 +0000 Subject: [PATCH 043/100] feat(benchmark): qualify Nemotron VL reranking Signed-off-by: chaofengw --- .../benchmark/nemotron-rerank-vl-1b-v2.yaml | 49 ++++++ .../tests/benchmark/reranking_reference.py | 104 +++++++++++ tools/benchmark_qualification/accuracy.py | 165 ++++++++++++++++++ .../benchmarks/reranking_performance.yaml | 15 ++ .../stsbenchmark_reranking_parity.yaml | 20 +++ tools/tests/test_architecture.py | 2 + tools/tests/test_model_benchmark.py | 10 ++ 7 files changed, 365 insertions(+) create mode 100644 families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml create mode 100644 families/eagle_vlm/tests/benchmark/reranking_reference.py create mode 100644 tools/benchmark_qualification/benchmarks/reranking_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml diff --git a/families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml b/families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml new file mode 100644 index 0000000000..f2759dc3ad --- /dev/null +++ b/families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-rerank-vl-1b-v2 +candidate: + family: eagle_vlm + checkpoint: nvidia/llama-nemotron-rerank-vl-1b-v2 + revision: 9e95da054312436dfc703319dd2b793a3bee2465 + task: reranking + precision: fp16 + trust_remote_code: true + bundle: nemotron-rerank-vl-1b-v2.bundle + build: + max_sequence_length: 685 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: stsbenchmark-reranking-parity + benchmark: stsbenchmark_reranking_parity + samples: 10 + reference: + command: reranking_reference.py + precision: fp32 + gate: + max_score_abs_error: 0.05 + min_sample_pass_rate: 0.9 + +performance: + - name: rerank + benchmark: reranking_performance + operation: rerank + request: + query: 0-dimensional biomaterials show inductive properties. + documents: + - Nanotechnologies provide new ways to manipulate and track stem cells. + - Depression is associated with immune dysregulation. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-reranking + mode: hf-eager + reference_backend: hf_transformers + precision: fp16 + output_contract: reranking-order diff --git a/families/eagle_vlm/tests/benchmark/reranking_reference.py b/families/eagle_vlm/tests/benchmark/reranking_reference.py new file mode 100644 index 0000000000..28f292945d --- /dev/null +++ b/families/eagle_vlm/tests/benchmark/reranking_reference.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Nemotron VL reranking reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + + +def _model_directory(model: str, revision: str | None) -> Path: + from huggingface_hub import snapshot_download + + offline = ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=offline, + ) + ) + + +def _dtype(torch_module: Any, precision: str) -> Any: + try: + return { + "bf16": torch_module.bfloat16, + "fp16": torch_module.float16, + "fp32": torch_module.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModelForSequenceClassification, AutoProcessor + + model_dir = _model_directory(str(request["model"]), request.get("revision")) + processor = AutoProcessor.from_pretrained( + model_dir, + trust_remote_code=True, + max_input_tiles=6, + use_thumbnail=True, + rerank_max_length=8192, + ) + model = ( + AutoModelForSequenceClassification.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + ) + .eval() + .to("cuda") + ) + results = [] + with torch.inference_mode(): + for sample in request["samples"]: + documents = [str(value) for value in sample["documents"]] + examples = [ + { + "question": str(sample["query"]), + "doc_text": document, + "doc_image": "", + } + for document in documents + ] + encoded = processor.process_queries_documents_crossencoder(examples) + encoded = { + name: value.to("cuda") if hasattr(value, "to") else value + for name, value in encoded.items() + } + logits = model(**encoded).logits.float().cpu() + if logits.ndim == 2: + logits = logits[:, 0] if logits.shape[-1] == 1 else logits[:, -1] + scores = [float(value) for value in logits.reshape(-1).tolist()] + if len(scores) != len(documents): + raise RuntimeError("reranking reference returned the wrong score count") + results.append({"sample_id": str(sample["sample_id"]), "scores": scores}) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index a109832fb9..979bae7f4e 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -37,6 +37,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _text_generation_parity(case, context, dataset, output) elif metric_name == "embedding_vector_parity": result = _encoder_embedding_parity(case, context, dataset, output) + elif metric_name == "reranking_score_parity": + result = _reranking_score_parity(case, context, dataset, output) elif metric_name == "forecast_tensor_parity": result = _etth1(case, context, definition, dataset, output) elif metric_name == "image_classification_top1_parity": @@ -295,6 +297,169 @@ def _encoder_embedding_parity( } +def _reranking_score_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + samples = _sts_reranking_samples(dataset.path, sample_limit) + reference = configured.get("reference", {}) + gate = configured.get("gate", {}) + if not isinstance(reference, Mapping) or not isinstance(gate, Mapping): + raise QualificationError("reranking reference and gate must be objects") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("reranking reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"reranking reference runner is not family-owned: {runner}") + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json( + request_path, + { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "samples": samples, + }, + ) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"reranking Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(samples): + raise QualificationError("reranking reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"query": sample["query"], "documents": sample["documents"]}, + } + for sample in samples + ] + actual, bundle = _candidate_outputs(case, context, output, "rerank", candidate_requests) + maximum_error = float(gate.get("max_score_abs_error", 0.05)) + minimum_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if maximum_error < 0.0 or not math.isfinite(maximum_error): + raise QualificationError("max_score_abs_error must be finite and nonnegative") + if not 0.0 <= minimum_rate <= 1.0: + raise QualificationError("min_sample_pass_rate must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(samples, actual, expected, strict=True): + candidate_scores = _reranking_scores(candidate_sample, "candidate") + reference_scores = _reranking_scores(reference_sample, "reference") + if len(candidate_scores) != len(reference_scores) or len(reference_scores) < 2: + raise QualificationError("reranking reference and candidate score counts differ") + errors = [ + abs(left - right) + for left, right in zip(candidate_scores, reference_scores, strict=True) + ] + candidate_order = _reranking_order(candidate_scores) + reference_order = _reranking_order(reference_scores) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": max(errors) <= maximum_error and candidate_order == reference_order, + "max_score_abs_error": max(errors), + "candidate_order": candidate_order, + "reference_order": reference_order, + } + ) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = pass_rate >= minimum_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": pass_rate, + "max_score_abs_error": max(row["max_score_abs_error"] for row in rows), + }, + "gate": { + "max_score_abs_error": maximum_error, + "min_sample_pass_rate": minimum_rate, + }, + "samples": rows, + } + + +def _reranking_scores(summary: Mapping[str, Any], label: str) -> list[float]: + values = summary.get("scores") + if not isinstance(values, list) or not values: + raise QualificationError(f"{label} reranking output has no scores") + scores = [float(value) for value in values] + if not all(math.isfinite(value) for value in scores): + raise QualificationError(f"{label} reranking output has non-finite scores") + return scores + + +def _reranking_order(scores: Sequence[float]) -> list[int]: + return sorted(range(len(scores)), key=lambda index: (-scores[index], index)) + + +def _sts_reranking_samples(path: Path, count: int) -> list[dict[str, Any]]: + pairs = [] + with path.open(encoding="utf-8") as stream: + for dataset_index, line in enumerate(stream): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise QualificationError( + f"STSBenchmark row {dataset_index} is not valid JSON" + ) from error + if not isinstance(row, Mapping): + raise QualificationError(f"STSBenchmark row {dataset_index} must be an object") + sentence1 = row.get("sentence1") + sentence2 = row.get("sentence2") + if not all( + isinstance(value, str) and value.strip() for value in (sentence1, sentence2) + ): + raise QualificationError( + f"STSBenchmark row {dataset_index} must contain two sentences" + ) + pairs.append((str(sentence1).strip(), str(sentence2).strip())) + if len(pairs) == count * 2: + break + if len(pairs) != count * 2: + raise QualificationError( + f"STSBenchmark contains {len(pairs)} usable rows; {count * 2} required" + ) + return [ + { + "sample_id": f"stsbenchmark-rerank-{index:06d}", + "query": pairs[index * 2][0], + "documents": [pairs[index * 2][1], pairs[index * 2 + 1][1]], + } + for index in range(count) + ] + + def _sts_samples(path: Path, count: int, prefix: str) -> list[dict[str, Any]]: pairs: list[dict[str, Any]] = [] with path.open(encoding="utf-8") as stream: diff --git a/tools/benchmark_qualification/benchmarks/reranking_performance.yaml b/tools/benchmark_qualification/benchmarks/reranking_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/reranking_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml b/tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml new file mode 100644 index 0000000000..387117abb7 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: stsbenchmark-test + version: pre1093-mteb-stsbenchmark-test-jsonl + sha256: 7c8927eef1ac4e256cae9768afc76232f79c39d19cb05d16bef92e11cecbe5bb + source: + mode: manual + path: STSBenchmark/stsbenchmark_test.jsonl + instructions: https://huggingface.co/datasets/mteb/stsbenchmark-sts + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: sts_reranking_jsonl +metric: + name: reranking_score_parity diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 5042d7bd32..4d41db1309 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -530,8 +530,10 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", + "tools/benchmark_qualification/benchmarks/reranking_performance.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", + "tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 3b2eef0bcd..a5fc4f8441 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -528,6 +528,16 @@ def test_vision_language_text_distance_is_normalized() -> None: assert qualification_accuracy._normalized_edit_distance(left, "blue car") > 0.15 +def test_reranking_order_is_stable_and_score_validation_is_strict() -> None: + assert qualification_accuracy._reranking_order([0.7, 0.2, 0.7]) == [0, 2, 1] + assert qualification_accuracy._reranking_scores({"scores": [0.7, -0.2]}, "test") == [ + 0.7, + -0.2, + ] + with pytest.raises(QualificationError, match="non-finite"): + qualification_accuracy._reranking_scores({"scores": [float("nan")]}, "test") + + def test_semantic_segmentation_accuracy_compares_pixel_and_class_iou( tmp_path: Path, monkeypatch ) -> None: From c7543d589fe1fd81ff12b536757492f192c4eed4 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 21:30:03 +0000 Subject: [PATCH 044/100] feat(benchmark): qualify SAM3 segmentation Signed-off-by: chaofengw --- .../trtmc_benchmark/tests/test_perf_matrix.py | 33 +++ families/sam3/tests/benchmark/reference.py | 262 ++++++++++++++++++ families/sam3/tests/benchmark/sam3.yaml | 52 ++++ tools/benchmark_qualification/accuracy.py | 185 ++++++++++++- ...genette_text_prompted_instance_parity.yaml | 20 ++ ...ext_prompted_segmentation_performance.yaml | 15 + tools/perf_matrix.py | 95 +++++++ tools/tests/test_architecture.py | 2 + tools/tests/test_model_benchmark.py | 17 ++ 9 files changed, 675 insertions(+), 6 deletions(-) create mode 100644 families/sam3/tests/benchmark/reference.py create mode 100644 families/sam3/tests/benchmark/sam3.yaml create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 0fb1df2b13..3ffe2a62af 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -3246,6 +3246,39 @@ def test_prompted_mask_contract_matches_binary_semantics() -> None: assert perf._output_contract(entry, candidate, reference)[0] is False +def test_instance_mask_contract_checks_masks_boxes_and_scores() -> None: + entry = SimpleNamespace( + spec={ + "baseline": { + "output_contract": "instance-mask-parity", + "min_mask_iou": 0.7, + "min_box_iou": 0.9, + "max_score_abs_error": 0.05, + } + } + ) + value = { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [1, 0, 0, 1], + "iou_scores": [0.9], + "boxes": [[0.0, 0.0, 2.0, 2.0]], + "box_coordinates": "original_image_pixels_xyxy", + } + candidate = {"output_summary": value} + reference = {"output_summary": dict(value)} + + matched, reason, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert reason == "" + assert evidence["minimum_mask_iou"] == evidence["minimum_box_iou"] == 1.0 + reference["output_summary"]["iou_scores"] = [0.7] + assert perf._output_contract(entry, candidate, reference)[0] is False + + def test_vision_language_text_contract_allows_small_normalized_differences() -> None: entry = SimpleNamespace( spec={ diff --git a/families/sam3/tests/benchmark/reference.py b/families/sam3/tests/benchmark/reference.py new file mode 100644 index 0000000000..d2f2634141 --- /dev/null +++ b/families/sam3/tests/benchmark/reference.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers SAM3 reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import Sam3Model, Sam3Processor + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = Sam3Processor.from_pretrained(model_dir) + model = Sam3Model.from_pretrained(model_dir, torch_dtype=dtype).eval().to("cuda") + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + encoded = reference.processor(images=image, text=prompt, return_tensors="pt") + return { + name: value.to("cuda") if hasattr(value, "to") else value for name, value in encoded.items() + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any]) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + outputs = reference.model(**inputs) + result = reference.processor.post_process_instance_segmentation( + outputs, + threshold=0.5, + mask_threshold=0.5, + target_sizes=inputs["original_sizes"].cpu().tolist(), + )[0] + masks = result["masks"].detach().float().cpu() + scores = result["scores"].detach().float().cpu().reshape(-1) + boxes = result["boxes"].detach().float().cpu().reshape(-1, 4) + if masks.ndim != 3 or masks.shape[0] != scores.numel() or boxes.shape[0] != scores.numel(): + raise RuntimeError("SAM3 reference returned inconsistent masks, scores or boxes") + height, width = (int(value) for value in masks.shape[-2:]) + count = int(masks.shape[0]) + return { + "segmented_images": 1, + "generated_masks": count, + "num_masks": count, + "mask_pixels": int(masks.numel()), + "height": height, + "width": width, + "mask_kind": "binary", + "masks": masks.reshape(-1).tolist(), + "iou_scores": scores.tolist(), + "boxes": boxes.tolist(), + "box_coordinates": "original_image_pixels_xyxy", + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + results = [] + for sample in samples: + prompt = str(sample.get("label_name", "")).strip() + if not prompt: + raise ValueError("SAM3 Accuracy sample has no label_name prompt") + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append({"sample_id": str(sample["sample_id"]), **_infer(reference, inputs)}) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "sam3" or arguments.operation != "segment_prompted": + raise ValueError("SAM3 reference only supports sam3 segment_prompted") + if arguments.mode != "pytorch-eager": + raise ValueError("SAM3 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request["prompt"])) + for _ in range(arguments.warmup): + _infer(reference, inputs) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sam3/tests/benchmark/sam3.yaml b/families/sam3/tests/benchmark/sam3.yaml new file mode 100644 index 0000000000..8ef47618a8 --- /dev/null +++ b/families/sam3/tests/benchmark/sam3.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: sam3 +candidate: + family: sam3 + checkpoint: facebook/sam3 + revision: 3c879f39826c281e95690f02c7821c4de09afae7 + task: text_prompted_segmentation + precision: fp32 + bundle: sam3.bundle + build: {} + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-text-instance-parity + benchmark: imagenette_text_prompted_instance_parity + samples: 10 + request: {} + reference: + command: reference.py + precision: fp32 + gate: + min_mask_iou: 0.7 + min_mask_match_rate: 1.0 + min_box_iou: 0.95 + max_score_abs_error: 0.05 + min_sample_pass_rate: 0.9 + +performance: + - name: segment-prompted + benchmark: text_prompted_segmentation_performance + operation: segment_prompted + request: + image_path: ../data/test_img.jpeg + prompt: car + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: instance-mask-parity + min_mask_iou: 0.7 + min_box_iou: 0.95 + max_score_abs_error: 0.05 diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 979bae7f4e..ea9cb1fd95 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -51,6 +51,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _object_detection_parity(case, context, dataset, output) elif metric_name == "prompted_segmentation_parity": result = _prompted_segmentation_parity(case, context, dataset, output) + elif metric_name == "text_prompted_instance_segmentation_parity": + result = _text_prompted_instance_segmentation_parity(case, context, dataset, output) elif metric_name == "semantic_segmentation_parity": result = _semantic_segmentation_parity(case, context, dataset, output) elif metric_name == "vision_language_text_parity": @@ -900,12 +902,14 @@ def _image_parity_samples(dataset: Dataset, sample_limit: int, task: str) -> lis image = (root / relative).resolve() if root not in image.parents or not image.is_file(): raise QualificationError(f"{task} request {index} image is unavailable: {image}") - selected.append( - { - "sample_id": str(request.get("id") or f"sample-{index}"), - "image_path": str(image), - } - ) + sample = { + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + } + label_name = request.get("label_name") + if isinstance(label_name, str) and label_name.strip(): + sample["label_name"] = label_name.strip() + selected.append(sample) return selected @@ -1290,6 +1294,175 @@ def _prompted_segmentation_parity( } +def _instance_masks( + value: Mapping[str, Any], label: str +) -> tuple[int, int, list[list[bool]], list[float], list[list[float]]]: + height, width, masks = _binary_masks(value, label) + scores = value.get("iou_scores") + boxes = value.get("boxes") + if ( + not isinstance(scores, list) + or len(scores) != len(masks) + or not isinstance(boxes, list) + or len(boxes) != len(masks) + or value.get("box_coordinates") != "original_image_pixels_xyxy" + ): + raise QualificationError(f"{label} instance output is incomplete") + numeric_scores = [float(score) for score in scores] + numeric_boxes = [] + for box in boxes: + if not isinstance(box, list) or len(box) != 4: + raise QualificationError(f"{label} instance box is invalid") + numeric_boxes.append([float(coordinate) for coordinate in box]) + if not all( + math.isfinite(number) + for number in (*numeric_scores, *(value for box in numeric_boxes for value in box)) + ): + raise QualificationError(f"{label} instance scores or boxes are not finite") + return height, width, masks, numeric_scores, numeric_boxes + + +def _match_instances( + candidate: tuple[int, int, list[list[bool]], list[float], list[list[float]]], + reference: tuple[int, int, list[list[bool]], list[float], list[list[float]]], +) -> list[dict[str, float]]: + if candidate[:2] != reference[:2]: + return [] + matched_reference: set[int] = set() + matches = [] + for index, mask in enumerate(candidate[2]): + choices = [ + (_mask_iou(mask, expected), expected_index) + for expected_index, expected in enumerate(reference[2]) + if expected_index not in matched_reference + ] + if not choices: + continue + mask_iou, expected_index = max(choices) + matched_reference.add(expected_index) + matches.append( + { + "mask_iou": mask_iou, + "box_iou": _box_iou(candidate[4][index], reference[4][expected_index]), + "score_abs_error": abs(candidate[3][index] - reference[3][expected_index]), + } + ) + return matches + + +def _text_prompted_instance_segmentation_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "text-prompted-segmentation") + if any(not sample.get("label_name") for sample in selected): + raise QualificationError("text-prompted segmentation samples require label_name") + expected = _family_image_reference(case, context, output, selected) + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + "image_path": sample["image_path"], + "prompt": sample["label_name"], + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs( + case, context, output, "segment_prompted", candidate_requests + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("text-prompted segmentation gate must be an object") + minimum_mask_iou = float(gate.get("min_mask_iou", 0.7)) + minimum_match_rate = float(gate.get("min_mask_match_rate", 1.0)) + minimum_box_iou = float(gate.get("min_box_iou", 0.9)) + maximum_score_error = float(gate.get("max_score_abs_error", 0.05)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 + for value in ( + minimum_mask_iou, + minimum_match_rate, + minimum_box_iou, + maximum_score_error, + minimum_sample_rate, + ) + ): + raise QualificationError("text-prompted segmentation gates must be in [0, 1]") + + rows = [] + total_matches = total_masks = 0 + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate = _instance_masks(candidate_sample, "candidate") + reference = _instance_masks(reference_sample, "reference") + matches = _match_instances(candidate, reference) + denominator = max(len(candidate[2]), len(reference[2]), 1) + match_rate = len(matches) / denominator + minimum_sample_mask_iou = min( + (match["mask_iou"] for match in matches), default=0.0 + ) + minimum_sample_box_iou = min( + (match["box_iou"] for match in matches), default=0.0 + ) + maximum_sample_score_error = max( + (match["score_abs_error"] for match in matches), default=1.0 + ) + passed = ( + match_rate >= minimum_match_rate + and minimum_sample_mask_iou >= minimum_mask_iou + and minimum_sample_box_iou >= minimum_box_iou + and maximum_sample_score_error <= maximum_score_error + ) + rows.append( + { + "sample_id": sample["sample_id"], + "prompt": sample["label_name"], + "passed": passed, + "candidate_masks": len(candidate[2]), + "reference_masks": len(reference[2]), + "mask_match_rate": match_rate, + "min_mask_iou": minimum_sample_mask_iou, + "min_box_iou": minimum_sample_box_iou, + "max_score_abs_error": maximum_sample_score_error, + } + ) + total_matches += len(matches) + total_masks += denominator + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "mask_match_rate": total_matches / total_masks, + "min_mask_iou": min(row["min_mask_iou"] for row in rows), + "min_box_iou": min(row["min_box_iou"] for row in rows), + "max_score_abs_error": max(row["max_score_abs_error"] for row in rows), + }, + "gate": { + "min_mask_iou": minimum_mask_iou, + "min_mask_match_rate": minimum_match_rate, + "min_box_iou": minimum_box_iou, + "max_score_abs_error": maximum_score_error, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _normalized_answer(value: Any) -> str: return " ".join(str(value or "").casefold().split()).strip(".,!?;:'\"") diff --git a/tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml new file mode 100644 index 0000000000..984a8f7d64 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_label_prompt +metric: + name: text_prompted_instance_segmentation_parity diff --git a/tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml b/tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 44f9900dee..60f9d327d4 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -77,6 +77,7 @@ "regression-values", "generated-token-count", "image-features-shape", + "instance-mask-parity", "localization", "media-shape", "metric-geometry-shape", @@ -1382,6 +1383,8 @@ def _output_contract( return matched, "segmentation output shape differs" if not matched else "", None if contract == "prompted-mask-parity": return _prompted_mask_parity(entry, left, right) + if contract == "instance-mask-parity": + return _instance_mask_parity(entry, left, right) if contract == "detection-parity": return _detection_parity(entry, left, right) if contract == "classification-top-class": @@ -2130,6 +2133,98 @@ def _prompted_mask_parity( return True, "", evidence +def _instance_mask_values( + summary: Mapping[str, Any], +) -> tuple[int, int, list[list[bool]], list[float], list[list[float]]] | None: + masks = _prompted_mask_values(summary) + if masks is None: + return None + scores = summary.get("iou_scores") + boxes = summary.get("boxes") + if ( + not isinstance(scores, list) + or len(scores) != len(masks[2]) + or not isinstance(boxes, list) + or len(boxes) != len(masks[2]) + or summary.get("box_coordinates") != "original_image_pixels_xyxy" + ): + return None + try: + numeric_scores = [float(score) for score in scores] + numeric_boxes = [ + [float(coordinate) for coordinate in box] + for box in boxes + if isinstance(box, list) and len(box) == 4 + ] + except (TypeError, ValueError): + return None + if len(numeric_boxes) != len(boxes) or not all( + math.isfinite(number) + for number in (*numeric_scores, *(value for box in numeric_boxes for value in box)) + ): + return None + return *masks, numeric_scores, numeric_boxes + + +def _instance_mask_parity( + entry: ResolvedEntry, + left: Mapping[str, Any], + right: Mapping[str, Any], +) -> tuple[bool, str, dict[str, Any] | None]: + candidate = _instance_mask_values(left) + reference = _instance_mask_values(right) + if candidate is None or reference is None: + return False, "instance mask output is invalid", None + if candidate[:2] != reference[:2] or len(candidate[2]) != len(reference[2]): + return ( + False, + "instance mask shape or count differs", + { + "candidate_shape": [len(candidate[2]), *candidate[:2]], + "reference_shape": [len(reference[2]), *reference[:2]], + }, + ) + matched_reference: set[int] = set() + matches = [] + for index, mask in enumerate(candidate[2]): + choices = [ + (_binary_mask_iou(mask, expected), expected_index) + for expected_index, expected in enumerate(reference[2]) + if expected_index not in matched_reference + ] + mask_iou, expected_index = max(choices) + matched_reference.add(expected_index) + matches.append( + ( + mask_iou, + _box_iou(candidate[4][index], reference[4][expected_index]), + abs(candidate[3][index] - reference[3][expected_index]), + ) + ) + minimum_mask_iou = min(match[0] for match in matches) + minimum_box_iou = min(match[1] for match in matches) + maximum_score_error = max(match[2] for match in matches) + required_mask_iou = float(entry.spec["baseline"].get("min_mask_iou", 0.7)) + required_box_iou = float(entry.spec["baseline"].get("min_box_iou", 0.9)) + allowed_score_error = float(entry.spec["baseline"].get("max_score_abs_error", 0.05)) + evidence = { + "instances": len(matches), + "minimum_mask_iou": minimum_mask_iou, + "required_mask_iou": required_mask_iou, + "minimum_box_iou": minimum_box_iou, + "required_box_iou": required_box_iou, + "maximum_score_abs_error": maximum_score_error, + "allowed_score_abs_error": allowed_score_error, + } + if minimum_mask_iou < required_mask_iou: + return False, "instance mask IoU is below the contract", evidence + if minimum_box_iou < required_box_iou: + return False, "instance box IoU is below the contract", evidence + if maximum_score_error > allowed_score_error: + return False, "instance score error exceeds the contract", evidence + return True, "", evidence + + def _detection_parity( entry: ResolvedEntry, left: Mapping[str, Any], diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 4d41db1309..ef48e6ee21 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -526,6 +526,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", @@ -537,6 +538,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", + "tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml", "tools/benchmark_qualification/benchmarks/vision_language_performance.yaml", "tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index a5fc4f8441..4999eb010e 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -520,6 +520,23 @@ def test_prompted_segmentation_masks_match_independent_of_order() -> None: assert qualification_accuracy._match_masks(candidate, reference) == [1.0, 1.0] +def test_text_prompted_instances_match_masks_boxes_and_scores() -> None: + summary = { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [1, 0, 0, 1], + "iou_scores": [0.9], + "boxes": [[0.0, 0.0, 2.0, 2.0]], + "box_coordinates": "original_image_pixels_xyxy", + } + parsed = qualification_accuracy._instance_masks(summary, "test") + matches = qualification_accuracy._match_instances(parsed, parsed) + + assert matches == [{"mask_iou": 1.0, "box_iou": 1.0, "score_abs_error": 0.0}] + + def test_vision_language_text_distance_is_normalized() -> None: left = qualification_accuracy._normalized_answer(" Red\ncar ") right = qualification_accuracy._normalized_answer("red car") From 473b0952d80d547893760a5c2e5eed6a6496f08d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 21:47:50 +0000 Subject: [PATCH 045/100] feat(benchmark): qualify K2 Horizon Uno Signed-off-by: chaofengw --- .../tests/test_family_performance.py | 21 ++++++ .../tests/benchmark/k2-horizon-7b-uno.yaml | 69 +++++++++++++++++++ tools/benchmark_qualification/accuracy.py | 21 +++++- tools/perf_matrix.py | 25 ++++++- tools/tests/test_model_benchmark.py | 8 ++- 5 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml diff --git a/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py b/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py index 4e73e48de7..e959c40910 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py @@ -761,6 +761,27 @@ def test_existing_adapter_command_keeps_existing_arguments(repository): assert command[command.index("--manifest") + 1] == str(entry.model.manifest_path) +def test_hf_reference_can_use_an_independent_pinned_model(repository): + entry = _resolved(repository) + baseline = { + key: value for key, value in entry.spec["baseline"].items() if key != "script" + } + baseline.update( + runner="hf-transformers", + model="fixture/reference", + revision="b" * 40, + ) + entry = replace(entry, spec={**entry.spec, "baseline": baseline}) + + environment = SimpleNamespace( + **vars(repository.environment), reference_python=Path(sys.executable) + ) + command = perf.baseline_command(entry, environment, repository.root / "out.json") + + assert command[command.index("--model") + 1] == "fixture/reference" + assert command[command.index("--revision") + 1] == "b" * 40 + + @pytest.mark.parametrize("failure", [None, "process", "identity"]) def test_execute_entry_validates_actual_script_process(repository, tmp_path, monkeypatch, failure): entry = _resolved(repository) diff --git a/families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml b/families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml new file mode 100644 index 0000000000..9e10a2074e --- /dev/null +++ b/families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: k2-horizon-7b-uno +candidate: + family: k2_horizon_uno + checkpoint: IFM/K2-Horizon-7B-Uno + revision: ec92bbd768f4a404319625204544782e3377bcd7 + task: text_generation + precision: bf16 + trust_remote_code: true + bundle: k2-horizon-7b-uno.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + model: IFM/K2-Horizon-7B + revision: 586b03f0fd1fbbf2f13eeafc33749e95ae34dd10 + trust_remote_code: true + precision: bf16 + request: &speculative-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + generation_mode: linear_spec_lora + block_length: 8 + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-26 + benchmark: text_generation_performance + operation: generate + request: + <<: *speculative-generation + prompt: Reply with the word OK. + max_new_tokens: 26 + use_chat_template: true + enable_thinking: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + model: IFM/K2-Horizon-7B + revision: 586b03f0fd1fbbf2f13eeafc33749e95ae34dd10 + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index ea9cb1fd95..1bd99a2cb2 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -103,10 +103,25 @@ def _text_generation_parity( source_language_placement = reference.get("source_language_placement") if source_language_placement is not None: reference_generation["source_language_placement"] = source_language_placement + reference_model = reference.get("model", case.candidate["checkpoint"]) + if not isinstance(reference_model, str) or not reference_model: + raise QualificationError("accuracy.reference.model must be a non-empty string") + reference_revision = reference.get("revision") + if reference_revision is None and reference_model == case.candidate["checkpoint"]: + reference_revision = case.candidate.get("revision") + if reference_revision is not None and ( + not isinstance(reference_revision, str) or not reference_revision + ): + raise QualificationError("accuracy.reference.revision must be a non-empty string") + trust_remote_code = reference.get( + "trust_remote_code", case.candidate.get("trust_remote_code", False) + ) + if not isinstance(trust_remote_code, bool): + raise QualificationError("accuracy.reference.trust_remote_code must be a boolean") reference_request = { - "model": str(case.candidate["checkpoint"]), - "revision": case.candidate.get("revision"), - "trust_remote_code": bool(case.candidate.get("trust_remote_code", False)), + "model": reference_model, + "revision": reference_revision, + "trust_remote_code": trust_remote_code, "precision": str(reference.get("precision", "fp32")), "task": reference_task, "output_token_policy": output_token_policy, diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 60f9d327d4..e47c26e0a6 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -839,6 +839,24 @@ def _baseline_mode(baseline: Mapping[str, Any]) -> str: return str(baseline.get("mode", "reference" if "script" in baseline else "torch-compile")) +def _baseline_model(entry: ResolvedEntry, environment: Environment) -> str: + configured = entry.spec["baseline"].get("model") + if configured is None: + configured = _adapter_options(entry, environment).get("model_id", entry.model.hf_id) + if not isinstance(configured, str) or not configured: + raise PerfMatrixError(f"entry {entry.spec['id']} baseline model must be non-empty") + return configured + + +def _baseline_revision(entry: ResolvedEntry, model: str) -> str | None: + configured = entry.spec["baseline"].get("revision") + if configured is None and model == entry.model.hf_id: + configured = entry.model.hf_revision or None + if configured is not None and (not isinstance(configured, str) or not configured): + raise PerfMatrixError(f"entry {entry.spec['id']} baseline revision must be non-empty") + return configured + + def baseline_command(entry: ResolvedEntry, environment: Environment, output: Path) -> list[str]: return _baseline_command(entry, environment, output) @@ -858,9 +876,10 @@ def _baseline_command( if source_placement is not None: request_value["source_language_placement"] = source_placement request = json.dumps(request_value, ensure_ascii=True, separators=(",", ":")) + model = _baseline_model(entry, environment) common = [ "--model", - str(_adapter_options(entry, environment).get("model_id", entry.model.hf_id)), + model, "--request-json", request, "--precision", @@ -933,7 +952,7 @@ def _baseline_command( arguments.extend(("--testcase-name", entry.case.testcase_name)) if "script" in baseline or entry.case.selected_task is not None: arguments.extend(("--selected-task", _effective_task(entry))) - revision = entry.model.hf_revision + revision = _baseline_revision(entry, model) if revision: arguments.extend(("--revision", revision)) if bool(entry.manifest.get("trust_remote_code", False)): @@ -949,7 +968,7 @@ def _validate_script_result( expected = { "schema_version": "trtmc.perf-baseline/v1", "status": "completed", - "model": str(_adapter_options(entry, environment).get("model_id", entry.model.hf_id)), + "model": _baseline_model(entry, environment), "family": entry.model.family, "operation": str(entry.spec["operation"]), "case_name": str(entry.spec["id"]), diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 4999eb010e..35b3498be2 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -159,6 +159,9 @@ def test_accuracy_forwards_declared_reference_model_load_options( "truncation_side": "left", "reference": { "command": "reference.py", + "model": "example/reference-model", + "revision": "b" * 40, + "trust_remote_code": False, "precision": "fp16", "experts_implementation": "batched_mm", }, @@ -219,8 +222,9 @@ def reference(command, *_args, **_kwargs): result = qualification_accuracy.run_accuracy(case, context) assert result["status"] == "passed" - assert captured["revision"] == "a" * 40 - assert captured["trust_remote_code"] is True + assert captured["model"] == "example/reference-model" + assert captured["revision"] == "b" * 40 + assert captured["trust_remote_code"] is False assert captured["experts_implementation"] == "batched_mm" assert runner in [Path(argument) for argument in captured_command] From 3843c17f72ce4a61cc1794f01393c5bf2c04676c Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 22:04:39 +0000 Subject: [PATCH 046/100] feat(benchmark): qualify DeepSeek OCR Signed-off-by: chaofengw --- .../tests/benchmark/deepseek-ocr.yaml | 63 ++++++++ .../deepseek_ocr/tests/benchmark/reference.py | 96 ++++++++++++ tools/benchmark_qualification/accuracy.py | 139 ++++++++++++++++++ .../benchmarks/ocrbench_v2_parity.yaml | 20 +++ tools/tests/test_architecture.py | 5 +- tools/tests/test_model_benchmark.py | 118 +++++++++++++++ 6 files changed, 439 insertions(+), 2 deletions(-) create mode 100644 families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml create mode 100644 families/deepseek_ocr/tests/benchmark/reference.py create mode 100644 tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml diff --git a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml new file mode 100644 index 0000000000..59c485676e --- /dev/null +++ b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deepseek-ocr +candidate: + family: deepseek_ocr + checkpoint: deepseek-ai/DeepSeek-OCR-2 + revision: aaa02f3811945a91062062994c5c4a3f4c0af2b0 + task: vision_language_generation + precision: fp16 + trust_remote_code: true + bundle: deepseek-ocr.bundle + build: + max_sequence_length: 4096 + fp32_layers: [6, 7, 8, 9, 10, 11, 12] + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: ocrbench-v2-parity + benchmark: ocrbench_v2_parity + samples: 10 + request: + max_new_tokens: 300 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: bf16 + gate: + max_normalized_edit_distance: 0.5 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + image_path: ../data/orc_test_img.jpeg + prompt: Extract the text from this image. + max_new_tokens: 300 + temperature: 1.0 + top_k: 1 + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-vlm + mode: hf-eager + reference_backend: hf_transformers + precision: bf16 + output_contract: ocr-text + max_normalized_edit_distance: 0.5 + required_substrings: + - Architecture + - "Attention:Standard Q/K/V/O" + - "RoPE:Standard rotary position embeddings" + - "Layer 0:Dense SwiGLU MLP" + - "Layers 1-11:MoE" + - "Vision: SAM ViT-B + Qwen2 encoder" diff --git a/families/deepseek_ocr/tests/benchmark/reference.py b/families/deepseek_ocr/tests/benchmark/reference.py new file mode 100644 index 0000000000..a84192ae1f --- /dev/null +++ b/families/deepseek_ocr/tests/benchmark/reference.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official DeepSeek OCR reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import tempfile + + +def _model_directory(model: str, revision: str | None) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ), + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModel, AutoTokenizer + + if not torch.cuda.is_available(): + raise RuntimeError("DeepSeek OCR Accuracy reference requires CUDA") + if request.get("precision") != "bf16": + raise ValueError("DeepSeek OCR official reference requires bf16") + model_dir = _model_directory(str(request["model"]), request.get("revision")) + tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) + model = ( + AutoModel.from_pretrained( + model_dir, + trust_remote_code=True, + use_safetensors=True, + torch_dtype=torch.bfloat16, + attn_implementation="eager", + ) + .to("cuda") + .eval() + ) + controls = request.get("request", {}) + if not isinstance(controls, dict): + raise ValueError("reference request controls must be an object") + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + results = [] + with tempfile.TemporaryDirectory(prefix="deepseek-ocr-accuracy-") as output_dir: + for sample in samples: + prompt = str(sample["prompt"]) + if "" not in prompt: + prompt = f"\n{prompt}" + text = model.infer( + tokenizer, + prompt=prompt, + image_file=str(sample["image_path"]), + output_path=output_dir, + base_size=1024, + image_size=768, + crop_mode=True, + save_results=False, + eval_mode=True, + ) + results.append( + { + "sample_id": str(sample["sample_id"]), + "text": str(text or ""), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 1bd99a2cb2..ec5ee70c2d 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -57,6 +57,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _semantic_segmentation_parity(case, context, dataset, output) elif metric_name == "vision_language_text_parity": result = _vision_language_text_parity(case, context, dataset, output) + elif metric_name == "ocr_text_parity": + result = _ocr_text_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -1569,6 +1571,143 @@ def _vision_language_text_parity( } +def _ocr_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("OCR dataset is not valid JSON") from error + samples = payload.get("samples") if isinstance(payload, Mapping) else None + if not isinstance(samples, list) or len(samples) < sample_limit: + raise QualificationError(f"OCR dataset requires at least {sample_limit} samples") + root = dataset.path.parent.resolve() + selected = [] + for index, sample in enumerate(samples[:sample_limit]): + if not isinstance(sample, Mapping): + raise QualificationError(f"OCR sample {index} must be an object") + media = sample.get("media") + image_value = None + if isinstance(media, list): + for item in media: + if isinstance(item, Mapping) and item.get("type") == "image": + image_value = item.get("path") + break + prompt = sample.get("question") + answer = sample.get("answer") + if not isinstance(image_value, str) or not image_value: + raise QualificationError(f"OCR sample {index} has no image") + if not isinstance(prompt, str) or not prompt.strip(): + raise QualificationError(f"OCR sample {index} has no question") + if not isinstance(answer, Mapping): + raise QualificationError(f"OCR sample {index} has no answer") + aliases = answer.get("aliases", []) + primary = answer.get("primary") + if not isinstance(aliases, list) or not all( + isinstance(value, str) and value.strip() for value in aliases + ): + raise QualificationError(f"OCR sample {index} has invalid answer aliases") + answers = [str(value).strip() for value in aliases] + if isinstance(primary, str) and primary.strip() and primary.strip() not in answers: + answers.insert(0, primary.strip()) + if not answers: + raise QualificationError(f"OCR sample {index} has no usable answer") + image = (root / image_value).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError(f"OCR sample {index} image is unavailable: {image}") + selected.append( + { + "sample_id": str(sample.get("id") or f"sample-{index}"), + "image_path": str(image), + "prompt": prompt.strip(), + "answers": answers, + } + ) + return selected + + +def _ocr_gold_match(text: str, answers: Sequence[str]) -> bool: + normalized = _normalized_answer(text).strip(".,!?;:'\"") + return any( + normalized == _normalized_answer(answer).strip(".,!?;:'\"") + for answer in answers + ) + + +def _ocr_text_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _ocr_samples(dataset, sample_limit) + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("OCR request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + **dict(request), + "image_path": sample["image_path"], + "prompt": sample["prompt"], + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("OCR gate must be an object") + maximum_distance = float(gate.get("max_normalized_edit_distance", 0.5)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= maximum_distance <= 1.0 or not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("OCR gates must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_text = _normalized_answer(candidate_sample.get("text")) + reference_text = _normalized_answer(reference_sample.get("text")) + distance = _normalized_edit_distance(candidate_text, reference_text) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": bool(candidate_text) and distance <= maximum_distance, + "candidate_text": candidate_text, + "reference_text": reference_text, + "normalized_edit_distance": distance, + "candidate_gold_match": _ocr_gold_match(candidate_text, sample["answers"]), + "reference_gold_match": _ocr_gold_match(reference_text, sample["answers"]), + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + candidate_gold_rate = sum(bool(row["candidate_gold_match"]) for row in rows) / len(rows) + reference_gold_rate = sum(bool(row["reference_gold_match"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if sample_pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "max_normalized_edit_distance": max(row["normalized_edit_distance"] for row in rows), + "candidate_normalized_gold_match_rate": candidate_gold_rate, + "reference_normalized_gold_match_rate": reference_gold_rate, + }, + "gate": { + "max_normalized_edit_distance": maximum_distance, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _semantic_segmentation_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml b/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml new file mode 100644 index 0000000000..7b9602ce94 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: ocrbench-v2 + version: unified-task-eval-v1 + sha256: e62c2fbdfe5d02f11d6e48d37ae548c854a19dcb3985eaa7ba3ba51492d10a94 + source: + mode: manual + path: OCRBench_v2/unified/dataset.json + instructions: https://huggingface.co/datasets/ling99/OCRBench_v2 + license: MIT +selection: + method: first +input: + adapter: ocrbench_v2 +metric: + name: ocr_text_parity diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index ef48e6ee21..ddb1fc532b 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -530,8 +530,9 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", - "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", - "tools/benchmark_qualification/benchmarks/reranking_performance.yaml", + "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", + "tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml", + "tools/benchmark_qualification/benchmarks/reranking_performance.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 35b3498be2..bfdc739292 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -549,6 +549,124 @@ def test_vision_language_text_distance_is_normalized() -> None: assert qualification_accuracy._normalized_edit_distance(left, "blue car") > 0.15 +def test_ocr_accuracy_uses_dataset_questions_and_reports_gold_matches( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "OCRBench_v2/unified" + image_root = dataset_root / "images" + image_root.mkdir(parents=True) + for name in ("a.jpg", "b.jpg"): + (image_root / name).write_bytes(b"fixture") + dataset_path = dataset_root / "dataset.json" + dataset_path.write_text( + json.dumps( + { + "samples": [ + { + "id": "a", + "question": "What is the setting?", + "media": [{"type": "image", "path": "images/a.jpg"}], + "answer": {"primary": "enabled", "aliases": ["enabled", "on"]}, + }, + { + "id": "b", + "question": "Which application?", + "media": [{"type": "image", "path": "images/b.jpg"}], + "answer": {"primary": "Facebook", "aliases": ["Facebook"]}, + }, + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="deepseek-ocr", + family="deepseek_ocr", + name="ocrbench-v2-parity", + benchmark="ocrbench_v2_parity", + candidate={ + "family": "deepseek_ocr", + "checkpoint": "deepseek-ai/DeepSeek-OCR-2", + "revision": "a" * 40, + "task": "vision_language_generation", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 2, + "request": {"max_new_tokens": 16}, + "reference": {"command": "reference.py", "precision": "bf16"}, + "gate": {"max_normalized_edit_distance": 0.1, "min_sample_pass_rate": 1.0}, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("ocrbench-v2", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path, + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured_reference: dict[str, object] = {} + captured_candidate: list[dict[str, object]] = [] + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured_reference.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "a", "text": "enabled"}, + {"sample_id": "b", "text": "Facebook"}, + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def candidate(_case, _context, _output, _operation, requests): + captured_candidate.extend(requests) + return ([{"text": "enabled"}, {"text": "facebook"}], tmp_path / "model.bundle") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "ocr_text_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr(qualification_accuracy, "_candidate_outputs", candidate) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["candidate_normalized_gold_match_rate"] == 1.0 + assert result["metrics"]["reference_normalized_gold_match_rate"] == 1.0 + assert captured_reference["samples"][0]["prompt"] == "What is the setting?" + assert captured_candidate[1]["request"]["prompt"] == "Which application?" + + def test_reranking_order_is_stable_and_score_validation_is_strict() -> None: assert qualification_accuracy._reranking_order([0.7, 0.2, 0.7]) == [0, 2, 1] assert qualification_accuracy._reranking_scores({"scores": [0.7, -0.2]}, "test") == [ From 5859fc3dfe97121bc6918b18f3c33b18f8c96a2b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 22:14:32 +0000 Subject: [PATCH 047/100] feat(benchmark): qualify LocateAnything Signed-off-by: chaofengw --- .../tests/benchmark/locateanything-3b.yaml | 60 +++++++ .../tests/benchmark/reference.py | 132 +++++++++++++++ tools/benchmark_qualification/accuracy.py | 151 ++++++++++++++++++ .../imagenette_localization_parity.yaml | 20 +++ tools/tests/test_architecture.py | 5 +- tools/tests/test_model_benchmark.py | 26 +++ 6 files changed, 392 insertions(+), 2 deletions(-) create mode 100644 families/locateanything/tests/benchmark/locateanything-3b.yaml create mode 100644 families/locateanything/tests/benchmark/reference.py create mode 100644 tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml diff --git a/families/locateanything/tests/benchmark/locateanything-3b.yaml b/families/locateanything/tests/benchmark/locateanything-3b.yaml new file mode 100644 index 0000000000..2c6f69558d --- /dev/null +++ b/families/locateanything/tests/benchmark/locateanything-3b.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: locateanything-3b +candidate: + family: locateanything + checkpoint: nvidia/LocateAnything-3B + revision: c32291ca5e996f5a7a485845b4f57a233936bba0 + task: vision_language_generation + precision: fp16 + trust_remote_code: true + bundle: locateanything-3b.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-localization-parity + benchmark: imagenette_localization_parity + samples: 10 + request: + prompt_template: "Locate a single instance that matches the following description: {label}." + max_new_tokens: 32 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + min_localization_box_iou: 0.9 + max_localization_point_distance: 10.0 + max_normalized_edit_distance: 0.5 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + image_path: ../data/test_img.jpeg + prompt: "Locate a single instance that matches the following description: white vehicle." + max_new_tokens: 32 + temperature: 1.0 + top_k: 1 + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-vlm + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: localization + min_localization_box_iou: 0.9 + max_localization_point_distance: 10.0 + max_normalized_edit_distance: 0.5 diff --git a/families/locateanything/tests/benchmark/reference.py b/families/locateanything/tests/benchmark/reference.py new file mode 100644 index 0000000000..5b4b8669c1 --- /dev/null +++ b/families/locateanything/tests/benchmark/reference.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official LocateAnything reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +from families.locateanything.tests.hf_reference import ( + _LocalTokenizer, + _load_config, + _repair_rotary_buffers, + manual_chat_prompt, +) +from families.locateanything.tests.vision_oracle import preprocess_image_inputs_for_trt + + +def _model_directory(model: str, revision: str | None) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ), + ) + ) + + +def _infer(model, tokenizer, image_path: str, prompt: str, max_new_tokens: int) -> str: + import torch + + image_inputs = preprocess_image_inputs_for_trt( + Path(image_path), + fixed_image_size=448, + patch_size=14, + image_mean=(0.5, 0.5, 0.5), + image_std=(0.5, 0.5, 0.5), + interpolation="bicubic", + ) + pixel_values = torch.from_numpy(image_inputs["pixel_values"]).to("cuda") + image_grid_hws = torch.from_numpy(image_inputs["image_grid_hws"]).to( + device="cuda", dtype=torch.int32 + ) + inputs = tokenizer(manual_chat_prompt(prompt), return_tensors="pt") + input_ids = inputs["input_ids"].to("cuda") + attention_mask = inputs["attention_mask"].to("cuda") + with torch.inference_mode(): + output = model.generate( + pixel_values=pixel_values, + image_grid_hws=image_grid_hws, + input_ids=input_ids, + attention_mask=attention_mask, + tokenizer=tokenizer, + max_new_tokens=max_new_tokens, + use_cache=True, + generation_mode="slow", + do_sample=False, + ) + if isinstance(output, str): + text = output + elif isinstance(output, (list, tuple)) and output and isinstance(output[0], str): + text = output[0] + else: + token_ids = output[0] if output.ndim > 1 else output + if token_ids.numel() > input_ids.shape[-1]: + token_ids = token_ids[input_ids.shape[-1] :] + text = tokenizer.decode(token_ids, skip_special_tokens=True) + return str(text).strip() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModel + + if not torch.cuda.is_available(): + raise RuntimeError("LocateAnything Accuracy reference requires CUDA") + if request.get("precision") != "fp32": + raise ValueError("LocateAnything official reference requires fp32") + model_dir = _model_directory(str(request["model"]), request.get("revision")) + config = _load_config(model_dir) + tokenizer = _LocalTokenizer(model_dir) + model = ( + AutoModel.from_pretrained( + model_dir, + config=config, + trust_remote_code=True, + local_files_only=True, + torch_dtype=torch.float32, + ) + .to("cuda") + .eval() + ) + _repair_rotary_buffers(model) + controls = request.get("request", {}) + prompt_template = controls.get("prompt_template") if isinstance(controls, dict) else None + if not isinstance(prompt_template, str) or "{label}" not in prompt_template: + raise ValueError("reference request.prompt_template must contain {label}") + max_new_tokens = int(controls.get("max_new_tokens", 32)) + results = [] + for sample in request.get("samples", []): + prompt = prompt_template.format(label=sample["label_name"]) + text = _infer(model, tokenizer, str(sample["image_path"]), prompt, max_new_tokens) + if not text: + raise RuntimeError("LocateAnything reference produced empty text") + results.append({"sample_id": str(sample["sample_id"]), "text": text}) + if not results: + raise ValueError("reference request must contain samples") + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index ec5ee70c2d..8a67afbe36 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -6,6 +6,7 @@ from __future__ import annotations import csv +from itertools import permutations import json import math import random @@ -59,6 +60,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _vision_language_text_parity(case, context, dataset, output) elif metric_name == "ocr_text_parity": result = _ocr_text_parity(case, context, dataset, output) + elif metric_name == "localization_text_parity": + result = _localization_text_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) @@ -1708,6 +1711,154 @@ def _ocr_text_parity( } +_LOCALIZATION_GROUP = re.compile(r"<(box|point)>(.*?)", re.DOTALL) + + +def _localization_values(text: str) -> tuple[str, tuple[tuple[float, ...], ...]]: + if "" not in text or "" not in text: + raise QualificationError("localization output omitted its reference tag") + groups = _LOCALIZATION_GROUP.findall(text) + if not groups: + raise QualificationError("localization output omitted box or point coordinates") + kind = groups[0][0] + values = [] + for current, raw in groups: + coordinates = tuple(float(value) for value in re.findall(r"-?\d+(?:\.\d+)?", raw)) + expected_size = 4 if current == "box" else 2 + if current != kind or len(coordinates) != expected_size: + raise QualificationError("localization output mixes coordinate kinds or ranks") + if not all(math.isfinite(value) and 0.0 <= value <= 1000.0 for value in coordinates): + raise QualificationError("localization coordinates must be finite values in [0, 1000]") + if current == "box" and ( + coordinates[2] <= coordinates[0] or coordinates[3] <= coordinates[1] + ): + raise QualificationError("localization box coordinates are not ordered xyxy") + values.append(coordinates) + return kind, tuple(values) + + +def _localization_alignment( + candidate: tuple[tuple[float, ...], ...], + reference: tuple[tuple[float, ...], ...], + kind: str, +) -> float: + if len(candidate) != len(reference): + return 0.0 if kind == "box" else math.inf + if kind == "box": + return max( + min(_box_iou(left, right) for left, right in zip(candidate, ordering, strict=True)) + for ordering in permutations(reference) + ) + return min( + max(math.dist(left, right) for left, right in zip(candidate, ordering, strict=True)) + for ordering in permutations(reference) + ) + + +def _localization_text_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "localization") + if any("label_name" not in sample for sample in selected): + raise QualificationError("localization dataset samples require label_name") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("localization request must be an object") + prompt_template = request.get("prompt_template") + if not isinstance(prompt_template, str) or "{label}" not in prompt_template: + raise QualificationError("localization request.prompt_template must contain {label}") + controls = {name: value for name, value in request.items() if name != "prompt_template"} + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + **controls, + "image_path": sample["image_path"], + "prompt": prompt_template.format(label=sample["label_name"]), + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("localization gate must be an object") + minimum_box_iou = float(gate.get("min_localization_box_iou", 0.9)) + maximum_point_distance = float(gate.get("max_localization_point_distance", 10.0)) + maximum_text_distance = float(gate.get("max_normalized_edit_distance", 0.5)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 + for value in (minimum_box_iou, maximum_text_distance, minimum_sample_rate) + ) or maximum_point_distance < 0.0 or not math.isfinite(maximum_point_distance): + raise QualificationError("localization gates are outside their valid ranges") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_text = _normalized_answer(candidate_sample.get("text")) + reference_text = _normalized_answer(reference_sample.get("text")) + reference_kind, reference_values = _localization_values(reference_text) + try: + candidate_kind, candidate_values = _localization_values(candidate_text) + except QualificationError: + candidate_kind = "invalid" + candidate_values = () + same_contract = candidate_kind == reference_kind and bool(candidate_values) + alignment = ( + _localization_alignment(candidate_values, reference_values, candidate_kind) + if same_contract + else 0.0 + ) + text_distance = _normalized_edit_distance(candidate_text, reference_text) + localization_passed = ( + alignment >= minimum_box_iou + if candidate_kind == "box" + else alignment <= maximum_point_distance if candidate_kind == "point" else False + ) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": localization_passed and text_distance <= maximum_text_distance, + "kind": candidate_kind, + "candidate_count": len(candidate_values), + "reference_count": len(reference_values), + "localization_alignment": alignment, + "normalized_edit_distance": text_distance, + "candidate_text": candidate_text, + "reference_text": reference_text, + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if sample_pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "max_normalized_edit_distance": max(row["normalized_edit_distance"] for row in rows), + }, + "gate": { + "min_localization_box_iou": minimum_box_iou, + "max_localization_point_distance": maximum_point_distance, + "max_normalized_edit_distance": maximum_text_distance, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _semantic_segmentation_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml new file mode 100644 index 0000000000..78bf6c8ea8 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_label_prompt +metric: + name: localization_text_parity diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index ddb1fc532b..ed264fc2ff 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -523,8 +523,9 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", "tools/benchmark_qualification/benchmarks/image_features_performance.yaml", "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml", + "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index bfdc739292..4d974d4f2e 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -667,6 +667,32 @@ def candidate(_case, _context, _output, _operation, requests): assert captured_candidate[1]["request"]["prompt"] == "Which application?" +def test_localization_accuracy_matches_unordered_boxes_and_points() -> None: + first_kind, first_boxes = qualification_accuracy._localization_values( + "cars<10><20><100><200><300><400><500><600>" + ) + second_kind, second_boxes = qualification_accuracy._localization_values( + "cars<300><400><500><600><10><20><100><200>" + ) + first_point_kind, first_points = qualification_accuracy._localization_values( + "cars<100><200>" + ) + second_point_kind, second_points = qualification_accuracy._localization_values( + "cars<106><208>" + ) + + assert first_kind == second_kind == "box" + assert qualification_accuracy._localization_alignment( + first_boxes, second_boxes, "box" + ) == 1.0 + assert first_point_kind == second_point_kind == "point" + assert qualification_accuracy._localization_alignment( + first_points, second_points, "point" + ) == 10.0 + with pytest.raises(QualificationError, match="reference tag"): + qualification_accuracy._localization_values("<10><20><100><200>") + + def test_reranking_order_is_stable_and_score_validation_is_strict() -> None: assert qualification_accuracy._reranking_order([0.7, 0.2, 0.7]) == [0, 2, 1] assert qualification_accuracy._reranking_scores({"scores": [0.7, -0.2]}, "test") == [ From 20759e23164bff435845f338e88c575bc7c9b082 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 22:26:08 +0000 Subject: [PATCH 048/100] fix(benchmark): run vision qualification inputs Signed-off-by: chaofengw --- apps/benchmark/trtmc_benchmark/task_adapters.py | 2 +- apps/benchmark/trtmc_benchmark/tests/test_benchmark.py | 9 +++++++++ families/yolo11/requirements.txt | 1 + families/yolov10/requirements.txt | 1 + families/yolov5/requirements.txt | 1 + families/yolov8/requirements.txt | 1 + 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/benchmark/trtmc_benchmark/task_adapters.py b/apps/benchmark/trtmc_benchmark/task_adapters.py index 22600ffc9d..054c756009 100644 --- a/apps/benchmark/trtmc_benchmark/task_adapters.py +++ b/apps/benchmark/trtmc_benchmark/task_adapters.py @@ -988,7 +988,7 @@ def _prompt(case: Mapping[str, Any], root: Path) -> str: def _image_path(case: Mapping[str, Any], root: Path) -> str: - return _required_asset(case, ("test_image", "image"), root, "image") + return _required_asset(case, ("image_path", "test_image", "image"), root, "image") def _audio_path(case: Mapping[str, Any], root: Path) -> str: diff --git a/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py b/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py index 959d31033d..101e33face 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py @@ -350,6 +350,15 @@ def test_image_only_detection_has_no_text_or_postprocessing_defaults(sdk_assets: assert metrics["images_per_s"] == 50.0 and metrics["detections_per_s"] == 0.0 +def test_legacy_image_task_accepts_canonical_image_path(sdk_assets: Path) -> None: + result = resolve_task_case("object_detection", {"image_path": "image.ppm"}, sdk_assets) + + assert result.request == { + "image_path": str(sdk_assets / "image.ppm"), + "batch_size": 1, + } + + def test_structure_input_keeps_document_bytes_and_source_path_opaque(tmp_path: Path) -> None: document = tmp_path / "prepared.bytes" document.write_bytes(b"B2RQ\0\x7f") diff --git a/families/yolo11/requirements.txt b/families/yolo11/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolo11/requirements.txt +++ b/families/yolo11/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolov10/requirements.txt b/families/yolov10/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolov10/requirements.txt +++ b/families/yolov10/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolov5/requirements.txt b/families/yolov5/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolov5/requirements.txt +++ b/families/yolov5/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolov8/requirements.txt b/families/yolov8/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolov8/requirements.txt +++ b/families/yolov8/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 From b7a12b517ab5639efcc8943be7ec7c0cfebf8ff2 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 22:36:37 +0000 Subject: [PATCH 049/100] fix(benchmark): accept qualification reranking inputs Signed-off-by: chaofengw --- apps/benchmark/trtmc_benchmark/task_adapters.py | 7 +++++-- .../trtmc_benchmark/tests/test_benchmark.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/apps/benchmark/trtmc_benchmark/task_adapters.py b/apps/benchmark/trtmc_benchmark/task_adapters.py index 054c756009..598ec93013 100644 --- a/apps/benchmark/trtmc_benchmark/task_adapters.py +++ b/apps/benchmark/trtmc_benchmark/task_adapters.py @@ -408,8 +408,11 @@ def _request(task: str, case: Mapping[str, Any], root: Path) -> dict[str, Any]: return {"prompt": _prompt(case, root), "batch_size": 1} if task == "reranking": inputs = _inputs(case) - query = inputs.get("query", inputs.get("prompt", case.get("prompt"))) - documents = inputs.get("documents") + query = inputs.get( + "query", + inputs.get("prompt", case.get("query", case.get("prompt"))), + ) + documents = inputs.get("documents", case.get("documents")) if not isinstance(query, str) or not query: raise BenchmarkError("reranking testcase requires inputs.prompt or inputs.query") if not isinstance(documents, list) or not documents: diff --git a/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py b/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py index 101e33face..5960716926 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py @@ -359,6 +359,19 @@ def test_legacy_image_task_accepts_canonical_image_path(sdk_assets: Path) -> Non } +def test_legacy_reranking_accepts_canonical_query_and_documents(sdk_assets: Path) -> None: + result = resolve_task_case( + "reranking", + {"query": "question", "documents": ["second", "first"]}, + sdk_assets, + ) + + assert result.request == { + "query": "question", + "documents": ["second", "first"], + } + + def test_structure_input_keeps_document_bytes_and_source_path_opaque(tmp_path: Path) -> None: document = tmp_path / "prepared.bytes" document.write_bytes(b"B2RQ\0\x7f") From 8aaa0a25fa3dc48465916aa24115169940c65728 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 22:45:29 +0000 Subject: [PATCH 050/100] fix(k2_horizon_uno): accept snapshot metadata files Signed-off-by: chaofengw --- families/k2_horizon_uno/support.py | 6 +++--- families/k2_horizon_uno/tests/test_model.py | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/families/k2_horizon_uno/support.py b/families/k2_horizon_uno/support.py index c0a7f4d8a4..e4373155da 100644 --- a/families/k2_horizon_uno/support.py +++ b/families/k2_horizon_uno/support.py @@ -7,10 +7,10 @@ _SUPPORT = FamilySupport(tasks=("text_generation",), default_task="text_generation") -_FILES = ("adapter_config.json", "adapter_model.safetensors") +_FILES = frozenset({"adapter_config.json", "adapter_model.safetensors"}) def describe(metadata: ModelMetadata) -> FamilySupport | None: - """Select the two-file adapter snapshot; build validates its exact recipe.""" + """Select snapshots containing the adapter pair; build validates the exact recipe.""" - return _SUPPORT if metadata.files == _FILES else None + return _SUPPORT if _FILES.issubset(metadata.files) else None diff --git a/families/k2_horizon_uno/tests/test_model.py b/families/k2_horizon_uno/tests/test_model.py index fcdaf6f911..d1c3ea45dc 100644 --- a/families/k2_horizon_uno/tests/test_model.py +++ b/families/k2_horizon_uno/tests/test_model.py @@ -92,7 +92,12 @@ def test_support_matches_only_the_adapter_identity() -> None: adapter = ModelMetadata({}, {}, ("adapter_config.json", "adapter_model.safetensors")) assert describe(adapter).tasks == ("text_generation",) assert describe(ModelMetadata({}, {}, ("adapter_config.json",))) is None - assert describe(ModelMetadata({}, {}, (*adapter.files, "README.md"))) is None + published = ModelMetadata( + {}, + {}, + (*adapter.files, ".gitattributes", "README.md", "conversion_summary.json"), + ) + assert describe(published).tasks == ("text_generation",) def test_exact_base_config_and_adapter_recipe(tmp_path: Path) -> None: From 4ed46d6fdf65fa3028acb5db95b31a316a90a78d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 23:04:40 +0000 Subject: [PATCH 051/100] fix(benchmark): run vision qualification end to end Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 3 +- .../native/test_benchmark_worker_e2e.cpp | 26 ++++ core/runtime/tests/fake_family.cpp | 11 ++ .../detr/tests/benchmark/detr-resnet-50.yaml | 2 +- .../tests/cpp/test_image_preprocess_seam.cpp | 14 ++ .../tests/benchmark/prepare_environment.py | 34 +++++ .../tests/benchmark/prepare_environment.py | 34 +++++ .../tests/benchmark/prepare_environment.py | 34 +++++ .../tests/benchmark/prepare_environment.py | 34 +++++ tools/benchmark_qualification/accuracy.py | 3 +- tools/benchmark_qualification/catalog.py | 4 + tools/benchmark_qualification/performance.py | 3 +- tools/benchmark_qualification/runtime.py | 34 ++++- tools/tests/test_model_benchmark.py | 142 ++++++++++++++++++ 14 files changed, 373 insertions(+), 5 deletions(-) create mode 100644 families/yolo11/tests/benchmark/prepare_environment.py create mode 100644 families/yolov10/tests/benchmark/prepare_environment.py create mode 100644 families/yolov5/tests/benchmark/prepare_environment.py create mode 100644 families/yolov8/tests/benchmark/prepare_environment.py diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index c4a552f524..510ebf0d8c 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -475,7 +475,8 @@ Json run_segment(trtmc::ITask& task, const Json& request, const Timing& timing) {"num_masks", 1}, {"height", result.height}, {"width", result.width}, - {"mask_pixels", result.mask.size()}}; + {"mask_pixels", result.mask.size()}, + {"mask", result.mask}}; }); } diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index fe197f3854..7d2275e340 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -159,6 +159,32 @@ int main(int argc, char** argv) { std::filesystem::remove(vector_bundle); } + const auto legacy_image = runtime_root / "benchmark_legacy_segment.ppm"; + { + std::ofstream image(legacy_image, std::ios::binary); + image << "P6\n2 1\n255\n"; + image.write("ABCDEF", 6); + } + const auto segment_bundle = runtime_root / "benchmark_fake_segmentation.bundle"; + write_bundle(segment_bundle, "segmentation"); + Json segment_request = request; + segment_request["case_name"] = "fake-segmentation"; + segment_request["bundle"] = segment_bundle.string(); + segment_request["operation"] = "segment"; + segment_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << segment_request << '\n'; + } + check(std::system(command.c_str()) == 0, "segmentation worker process completed"); + std::ifstream segment_output_file(output_path); + Json segment_result; + segment_output_file >> segment_result; + check(segment_result.at("output_summary").at("mask") == Json::array({7, 7}), + "legacy segmentation retains labels for Accuracy comparison"); + std::filesystem::remove(segment_bundle); + std::filesystem::remove(legacy_image); + Json sdk = request; sdk["runtime_root"] = argv[3]; sdk["operation"] = "generate"; diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 1e09a355e8..7c82c19fe0 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -56,6 +56,15 @@ class FakeEmbedding final : public trtmc::IEmbedding { } }; +class FakeSegmentation final : public trtmc::ISegmentation { + public: + trtmc::SegmentResult segment(const float*, std::int32_t height, + std::int32_t width) override { + return {std::vector(static_cast(height) * width, 7), height, + width}; + } +}; + } // namespace extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { @@ -73,6 +82,8 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context return new FakeEncoding(); if (context.reader.info().task == trtmc::IEmbedding::kTask) return new FakeEmbedding(); + if (context.reader.info().task == trtmc::ISegmentation::kTask) + return new FakeSegmentation(); if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) return new FakeForecast(context.backend, context.kv_cache_size_bytes); if (context.reader.info().task == trtmc::ITextGeneration::kTask) { diff --git a/families/detr/tests/benchmark/detr-resnet-50.yaml b/families/detr/tests/benchmark/detr-resnet-50.yaml index 471203b346..211aec65a1 100644 --- a/families/detr/tests/benchmark/detr-resnet-50.yaml +++ b/families/detr/tests/benchmark/detr-resnet-50.yaml @@ -12,7 +12,7 @@ candidate: bundle: detr-resnet-50.bundle build: max_sequence_length: 1 - image_height: 796 + image_height: 1333 image_width: 1333 reference_environment: diff --git a/families/detr/tests/cpp/test_image_preprocess_seam.cpp b/families/detr/tests/cpp/test_image_preprocess_seam.cpp index 64a29e7891..23182fb35f 100644 --- a/families/detr/tests/cpp/test_image_preprocess_seam.cpp +++ b/families/detr/tests/cpp/test_image_preprocess_seam.cpp @@ -86,6 +86,19 @@ void test_detr_preprocess_fits_half_tie_engine_dimensions() { check(portrait.size() == 3U * 1333U * 666U, "detr portrait fits reference dimensions"); } +void test_detr_square_engine_accepts_both_image_orientations() { + trtmc::DetrPreprocessConfig config; + config.input_image_h = 1333; + config.input_image_w = 1333; + + const auto landscape = trtmc::compute_detr_resize_shape(382, 640, config); + const auto portrait = trtmc::compute_detr_resize_shape(640, 382, config); + check(landscape.height == 796 && landscape.width == 1333, + "detr square engine accepts landscape resize"); + check(portrait.height == 1333 && portrait.width == 796, + "detr square engine accepts portrait resize"); +} + void test_detr_preprocess_applies_normalization() { const std::vector pixels(3U * 2U * 2U, 0.75F); trtmc::DetrPreprocessConfig config; @@ -160,6 +173,7 @@ int main() { test_detr_resize_rounds_half_ties_to_even(); test_detr_resize_rounds_non_ties_to_nearest(); test_detr_preprocess_fits_half_tie_engine_dimensions(); + test_detr_square_engine_accepts_both_image_orientations(); test_detr_preprocess_applies_normalization(); test_detr_preprocess_rejects_invalid_config(); test_detr_preprocess_rejects_resize_larger_than_engine_input(); diff --git a/families/yolo11/tests/benchmark/prepare_environment.py b/families/yolo11/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolo11/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolov10/tests/benchmark/prepare_environment.py b/families/yolov10/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolov10/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolov5/tests/benchmark/prepare_environment.py b/families/yolov5/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolov5/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolov8/tests/benchmark/prepare_environment.py b/families/yolov8/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolov8/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 8a67afbe36..0c0d43aa9f 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -18,6 +18,7 @@ from .datasets import Dataset, resolve_dataset from .runtime import ( RuntimeContext, + benchmark_executable, prepare_bundle, reference_python, require_candidate, @@ -855,7 +856,7 @@ def _candidate_outputs( _json(data_path, list(requests)) candidate_output = output / "candidate-run" command = [ - str(context.trtmc_bench), + str(benchmark_executable(case, context)), "run", "--model", str(descriptor), diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 0515dfc8da..5e2859680d 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -30,6 +30,7 @@ class QualificationCase: values: Mapping[str, Any] source: Path reference_requirements: Path | None + environment_hook: Path | None = None @property def id(self) -> str: @@ -52,6 +53,8 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: raise QualificationError(f"{path}: file name must match model {model!r}") candidate = _candidate(raw.get("candidate"), family, path) requirements = _requirements(raw.get("reference_environment"), path) + hook_path = path.parent / "prepare_environment.py" + environment_hook = hook_path.resolve() if hook_path.is_file() else None for kind in ("accuracy", "performance"): configured = raw.get(kind, []) if not isinstance(configured, list): @@ -71,6 +74,7 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: values=dict(value), source=path.resolve(), reference_requirements=requirements, + environment_hook=environment_hook, ) if case.id in seen: raise QualificationError(f"duplicate qualification case {case.id!r}") diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py index 81acb638d9..566e0d00be 100644 --- a/tools/benchmark_qualification/performance.py +++ b/tools/benchmark_qualification/performance.py @@ -16,6 +16,7 @@ from .catalog import QualificationCase, QualificationError, load_benchmark from .runtime import ( RuntimeContext, + benchmark_executable, reference_python, require_candidate, run_command, @@ -128,7 +129,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st "schema_version": "trtmc.perf-environment/v2", "name": "qualification", "tools": { - "trtmc_bench": str(context.trtmc_bench), + "trtmc_bench": str(benchmark_executable(case, context)), "trtmc_worker": str(worker), "hf_transformers_runner": str( context.repository / "apps/benchmark/performance/baselines/hf_transformers.py" diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 64889649c8..58299e7c14 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -9,6 +9,7 @@ import html import json import os +import shlex import site import shutil import subprocess @@ -126,7 +127,7 @@ def prepare_bundle( descriptor: Path, ) -> Path: command = [ - str(context.trtmc_bench), + str(benchmark_executable(case, context)), "run", "--model", str(descriptor), @@ -167,8 +168,11 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: if requirements is None: return Path(os.path.abspath(sys.executable)) digest = hashlib.sha256() + digest.update(b"qualification-family-environment-v1\0") digest.update(requirements.read_bytes()) digest.update(sys.version.encode()) + if case.environment_hook is not None: + digest.update(case.environment_hook.read_bytes()) environment = context.environment_root / f"{case.family}-{digest.hexdigest()[:12]}" python = environment / "bin/python" stamp = environment / ".qualification-requirements.sha256" @@ -210,10 +214,38 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: ) if installed.returncode != 0: raise QualificationError(f"cannot install reference requirements; see {setup_root}") + if case.environment_hook is not None: + prepared = run_command( + [str(python), str(case.environment_hook)], + setup_root, + "prepare", + timeout=1800, + verbose=context.verbose, + ) + if prepared.returncode != 0: + raise QualificationError(f"cannot prepare family environment; see {setup_root}") stamp.write_text(expected + "\n", encoding="utf-8") return python +def benchmark_executable(case: QualificationCase, context: RuntimeContext) -> Path: + """Run user-facing benchmark and bundle build with the family Python environment.""" + python = reference_python(case, context) + digest = hashlib.sha256(f"{python.resolve()}\0{context.trtmc_bench}".encode()).hexdigest()[:12] + launcher = context.environment_root / "launchers" / f"{case.family}-{digest}" + contents = ( + "#!/bin/sh\n" + f"exec {shlex.quote(str(python))} {shlex.quote(str(context.trtmc_bench))} \"$@\"\n" + ) + if not launcher.is_file() or launcher.read_text(encoding="utf-8") != contents: + launcher.parent.mkdir(parents=True, exist_ok=True) + temporary = launcher.with_name(f".{launcher.name}.{os.getpid()}.tmp") + temporary.write_text(contents, encoding="utf-8") + temporary.chmod(0o755) + os.replace(temporary, launcher) + return launcher + + def _inherit_parent_site_packages(environment: Path) -> None: child_packages = sorted(environment.glob("lib/python*/site-packages")) if len(child_packages) != 1: diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 4d974d4f2e..87be3fa1ba 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -27,6 +27,7 @@ from tools.benchmark_qualification.references import hf_encoder, hf_text_generation from tools.benchmark_qualification.runtime import ( RuntimeContext, + benchmark_executable, prepare_bundle, reference_python, run_command, @@ -130,6 +131,31 @@ def test_one_model_file_owns_multiple_cases_without_testcase_indirection( assert case.candidate["checkpoint"] == "example/model" +def test_family_environment_hook_is_discovered_with_the_model_profiles(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + " task: text_generation\n" + " precision: fp16\n" + "accuracy:\n" + " - name: continuation\n" + " benchmark: example_accuracy\n" + "performance: []\n", + encoding="utf-8", + ) + hook = profile.parent / "prepare_environment.py" + hook.write_text("# family hook\n", encoding="utf-8") + + (case,) = discover(tmp_path) + + assert case.environment_hook == hook.resolve() + + def test_accuracy_forwards_declared_reference_model_load_options( tmp_path: Path, monkeypatch, @@ -1822,6 +1848,122 @@ def complete(command, *_args, **_kwargs): assert inherited.read_text(encoding="utf-8") == f"{parent_packages.resolve()}\n" +def test_family_environment_hook_runs_after_requirements_install( + tmp_path: Path, monkeypatch +) -> None: + requirements = tmp_path / "requirements.txt" + requirements.write_text("example==1\n", encoding="utf-8") + hook = tmp_path / "prepare_environment.py" + hook.write_text("# family hook\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="custom-environment-model", + family="example", + name="continuation", + benchmark="mmlu_continuation", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=requirements, + environment_hook=hook, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + commands: list[list[str]] = [] + parent_packages = tmp_path / "parent/site-packages" + parent_packages.mkdir(parents=True) + + def complete(command, *_args, **_kwargs): + commands.append(list(command)) + if command[1:3] == ["-m", "venv"]: + environment = Path(command[-1]) + (environment / "bin").mkdir(parents=True) + (environment / "bin/python").write_text("", encoding="utf-8") + child = environment / ( + f"lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages" + ) + child.mkdir(parents=True) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr("tools.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("site.getsitepackages", lambda: [str(parent_packages)]) + + python = reference_python(case, context) + + assert commands[-1] == [str(python), str(hook)] + + +def test_benchmark_launcher_uses_the_family_python(tmp_path: Path, monkeypatch) -> None: + python = tmp_path / "family env/bin/python" + python.parent.mkdir(parents=True) + python.write_text("", encoding="utf-8") + bench = tmp_path / "user tools/trtmc-bench" + bench.parent.mkdir(parents=True) + bench.write_text("", encoding="utf-8") + case = QualificationCase( + kind="performance", + model="example", + family="example", + name="generate", + benchmark="text_generation_performance", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=bench, + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + monkeypatch.setattr( + "tools.benchmark_qualification.runtime.reference_python", lambda *_args: python + ) + + launcher = benchmark_executable(case, context) + + assert os.access(launcher, os.X_OK) + assert launcher.read_text(encoding="utf-8") == ( + "#!/bin/sh\n" + f"exec '{python}' '{bench}' \"$@\"\n" + ) + + def test_bundle_preparation_uses_the_selected_runtime(tmp_path: Path, monkeypatch) -> None: case = QualificationCase( kind="accuracy", From 36f49934fe85e4df831a00b80b4e0fb0489ed07d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 19:14:56 +0000 Subject: [PATCH 052/100] fix(benchmark): support extension build environments Allow family benchmark profiles to opt out of pip build isolation when reference extensions need packages from the inherited base environment. Use the option for Nemotron-H so mamba-ssm can import PyTorch while building. Signed-off-by: chaofengw --- .../tests/benchmark/nemotron-h-nano-9b.yaml | 1 + tools/benchmark_qualification/catalog.py | 19 ++++++++++++++++- tools/benchmark_qualification/runtime.py | 21 +++++++++++-------- tools/tests/test_model_benchmark.py | 7 +++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml b/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml index f21f84164c..0c163f4f01 100644 --- a/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml +++ b/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml @@ -16,6 +16,7 @@ candidate: reference_environment: requirements: ../../requirements.txt + build_isolation: false accuracy: - name: mmlu-continuation diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 5e2859680d..280923fe14 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -30,6 +30,7 @@ class QualificationCase: values: Mapping[str, Any] source: Path reference_requirements: Path | None + reference_build_isolation: bool = True environment_hook: Path | None = None @property @@ -52,7 +53,9 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: if path.stem != model: raise QualificationError(f"{path}: file name must match model {model!r}") candidate = _candidate(raw.get("candidate"), family, path) - requirements = _requirements(raw.get("reference_environment"), path) + reference_environment = raw.get("reference_environment") + requirements = _requirements(reference_environment, path) + build_isolation = _build_isolation(reference_environment, path) hook_path = path.parent / "prepare_environment.py" environment_hook = hook_path.resolve() if hook_path.is_file() else None for kind in ("accuracy", "performance"): @@ -74,6 +77,7 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: values=dict(value), source=path.resolve(), reference_requirements=requirements, + reference_build_isolation=build_isolation, environment_hook=environment_hook, ) if case.id in seen: @@ -163,6 +167,19 @@ def _requirements(raw: Any, path: Path) -> Path | None: return result +def _build_isolation(raw: Any, path: Path) -> bool: + if raw is None: + return True + if not isinstance(raw, Mapping): + raise QualificationError(f"{path}: reference_environment must be an object") + value = raw.get("build_isolation", True) + if not isinstance(value, bool): + raise QualificationError( + f"{path}: reference_environment.build_isolation must be a boolean" + ) + return value + + def _yaml_object(path: Path, label: str) -> dict[str, Any]: try: value = yaml.safe_load(path.read_text(encoding="utf-8")) diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 58299e7c14..7996590fb2 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -171,6 +171,7 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: digest.update(b"qualification-family-environment-v1\0") digest.update(requirements.read_bytes()) digest.update(sys.version.encode()) + digest.update(f"build-isolation={case.reference_build_isolation}".encode()) if case.environment_hook is not None: digest.update(case.environment_hook.read_bytes()) environment = context.environment_root / f"{case.family}-{digest.hexdigest()[:12]}" @@ -197,16 +198,18 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: if created.returncode != 0 or not python.is_file(): raise QualificationError(f"cannot create reference environment; see {setup_root}") _inherit_parent_site_packages(environment) + install_command = [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + ] + if not case.reference_build_isolation: + install_command.append("--no-build-isolation") + install_command.extend(("-r", str(requirements))) installed = run_command( - [ - str(python), - "-m", - "pip", - "install", - "--disable-pip-version-check", - "-r", - str(requirements), - ], + install_command, setup_root, "pip", timeout=1800, diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 87be3fa1ba..e9c96fb4f7 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -48,6 +48,8 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert relative.parts[1:3] == ("tests", "benchmark") assert case.kind in {"accuracy", "performance"} assert not any("l0" in case.model.lower() for case in cases) + nemotron_h = [case for case in cases if case.model == "nemotron-h-nano-9b"] + assert nemotron_h and all(not case.reference_build_isolation for case in nemotron_h) def test_l0_configs_outside_the_benchmark_folder_are_not_discovered( @@ -1806,6 +1808,7 @@ def test_family_reference_environment_inherits_parent_venv_packages( values={}, source=tmp_path / "example.yaml", reference_requirements=requirements, + reference_build_isolation=False, ) context = RuntimeContext( repository=REPOSITORY, @@ -1824,8 +1827,10 @@ def test_family_reference_environment_inherits_parent_venv_packages( ) parent_packages = tmp_path / "parent-venv/site-packages" parent_packages.mkdir(parents=True) + commands = [] def complete(command, *_args, **_kwargs): + commands.append(command) if command[1:3] == ["-m", "venv"]: environment = Path(command[-1]) (environment / "bin").mkdir(parents=True) @@ -1846,6 +1851,8 @@ def complete(command, *_args, **_kwargs): "trtmc-parent-environment.pth" ) assert inherited.read_text(encoding="utf-8") == f"{parent_packages.resolve()}\n" + pip_command = next(command for command in commands if command[1:3] == ["-m", "pip"]) + assert "--no-build-isolation" in pip_command def test_family_environment_hook_runs_after_requirements_install( From 9b19e3d9f80918a7b08d218dce21ad4861bde6ae Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 18:49:49 +0000 Subject: [PATCH 053/100] fix(benchmark): load PatchTSMixer references Signed-off-by: chaofengw --- .../patchtsmixer/tests/benchmark/reference.py | 10 +++++++++- .../tests/test_benchmark_reference.py | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 families/patchtsmixer/tests/test_benchmark_reference.py diff --git a/families/patchtsmixer/tests/benchmark/reference.py b/families/patchtsmixer/tests/benchmark/reference.py index 3626cb4478..aa4655c019 100644 --- a/families/patchtsmixer/tests/benchmark/reference.py +++ b/families/patchtsmixer/tests/benchmark/reference.py @@ -12,6 +12,13 @@ from pathlib import Path +def prediction_model_class(transformers): + model_class = transformers.PatchTSMixerForPrediction + if not hasattr(model_class, "all_tied_weights_keys"): + model_class.all_tied_weights_keys = {} + return model_class + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--request", required=True, type=Path) @@ -32,8 +39,9 @@ def main() -> int: if request.get("revision"): options["revision"] = request["revision"] config = transformers.AutoConfig.from_pretrained(str(request["model"]), **options) + model_class = prediction_model_class(transformers) model = ( - transformers.PatchTSMixerForPrediction.from_pretrained( + model_class.from_pretrained( str(request["model"]), torch_dtype=torch.float32, **options ) .eval() diff --git a/families/patchtsmixer/tests/test_benchmark_reference.py b/families/patchtsmixer/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..4e547c9cf0 --- /dev/null +++ b/families/patchtsmixer/tests/test_benchmark_reference.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from families.patchtsmixer.tests.benchmark.reference import prediction_model_class + + +def test_reference_adds_transformers_tied_weights_compatibility_field() -> None: + class PredictionModel: + pass + + assert not hasattr(PredictionModel, "all_tied_weights_keys") + + selected = prediction_model_class( + SimpleNamespace(PatchTSMixerForPrediction=PredictionModel) + ) + + assert selected is PredictionModel + assert selected.all_tied_weights_keys == {} From caeca450aa3688496cf1c6b82a889c07fdc483da Mon Sep 17 00:00:00 2001 From: chaofengw Date: Fri, 18 Sep 2026 23:48:45 +0000 Subject: [PATCH 054/100] feat(benchmark): qualify Fast Foundation Stereo Signed-off-by: chaofengw --- .../benchmark/fast-foundation-stereo.yaml | 57 ++++ .../tests/benchmark/prepare_environment.py | 81 ++++++ .../tests/benchmark/reference.py | 260 ++++++++++++++++++ .../tests/data/benchmark_stereo.json | 9 + tools/benchmark_qualification/accuracy.py | 219 ++++++++++++++- .../benchmarks/stereo_disparity_parity.yaml | 19 ++ .../stereo_disparity_performance.yaml | 15 + tools/benchmark_qualification/datasets.py | 20 +- tools/tests/test_model_benchmark.py | 167 +++++++++++ 9 files changed, 844 insertions(+), 3 deletions(-) create mode 100644 families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml create mode 100644 families/fast_foundation_stereo/tests/benchmark/prepare_environment.py create mode 100644 families/fast_foundation_stereo/tests/benchmark/reference.py create mode 100644 families/fast_foundation_stereo/tests/data/benchmark_stereo.json create mode 100644 tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml diff --git a/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml b/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml new file mode 100644 index 0000000000..daa88cfffb --- /dev/null +++ b/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: fast-foundation-stereo +candidate: + family: fast_foundation_stereo + checkpoint: nvidia/c-fast-foundationstereo + revision: 9b446878c81ddb27593036767b29b2859d46103e + task: stereo_disparity + precision: fp16 + bundle: fast-foundation-stereo.bundle + build: + tensor_parallel_size: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: office-disparity-parity + benchmark: stereo_disparity_parity + samples: 1 + request: &disparity-request + height: 700 + width: 700 + max_disp: 192 + valid_iters: 8 + reference: + command: reference.py + precision: fp16 + gate: + min_cosine: 0.999 + max_mean_abs_error: 0.5 + max_bad_2px_fraction: 0.03 + min_sample_pass_rate: 1.0 + +performance: + - name: disparity + benchmark: stereo_disparity_performance + operation: disparity + request: + <<: *disparity-request + left_image_path: ../data/office_left.png + right_image_path: ../data/office_right.png + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: official_pytorch + precision: fp16 + output_contract: disparity-parity + min_disparity_cosine: 0.999 + max_disparity_mean_abs_error: 0.5 + max_disparity_bad_2px_fraction: 0.03 diff --git a/families/fast_foundation_stereo/tests/benchmark/prepare_environment.py b/families/fast_foundation_stereo/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..0a875ee350 --- /dev/null +++ b/families/fast_foundation_stereo/tests/benchmark/prepare_environment.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned official reference implementation inside this family environment.""" + +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +from huggingface_hub import hf_hub_download + + +SOURCE_REPOSITORY = "https://github.com/NVlabs/Fast-FoundationStereo.git" +SOURCE_REVISION = "a290ba04c1b3ad1ec41a33974a157b2917b624d4" +CHECKPOINT = "nvidia/c-fast-foundationstereo" +CHECKPOINT_REVISION = "9b446878c81ddb27593036767b29b2859d46103e" + + +def main() -> int: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "Fast-FoundationStereo" + required = ( + destination / "core/foundation_stereo.py", + destination / "core/submodule.py", + destination / "weights/23-36-37/model_best_bp2_serialize.pth", + ) + if all(path.is_file() for path in required): + return 0 + if destination.exists(): + raise RuntimeError(f"incomplete Fast Foundation Stereo reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="Fast-FoundationStereo.", dir=parent)) + try: + subprocess.run( + ["git", "init", "--quiet", str(temporary)], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", SOURCE_REPOSITORY], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(temporary), + "fetch", + "--quiet", + "--depth=1", + "origin", + SOURCE_REVISION, + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + checkpoint = Path( + hf_hub_download( + repo_id=CHECKPOINT, + revision=CHECKPOINT_REVISION, + filename="model_best_bp2_serialize.pth", + ) + ) + target = temporary / "weights/23-36-37/model_best_bp2_serialize.pth" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(checkpoint, target) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/fast_foundation_stereo/tests/benchmark/reference.py b/families/fast_foundation_stereo/tests/benchmark/reference.py new file mode 100644 index 0000000000..5077df139b --- /dev/null +++ b/families/fast_foundation_stereo/tests/benchmark/reference.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned official Accuracy and Performance reference for Fast Foundation Stereo.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import statistics +import sys +import time +from typing import Any, Mapping + + +def _model_root() -> Path: + root = Path(sys.prefix) / "trtmc-reference/Fast-FoundationStereo" + required = ( + root / "core/foundation_stereo.py", + root / "core/submodule.py", + root / "weights/23-36-37/model_best_bp2_serialize.pth", + ) + if not all(path.is_file() for path in required): + raise RuntimeError(f"Fast Foundation Stereo reference is incomplete: {root}") + return root + + +def _load_model(max_disparity: int, valid_iters: int): + import torch + + root = _model_root() + previous = Path.cwd() + try: + os.chdir(root) + sys.path.insert(0, str(root)) + from core.utils.utils import InputPadder + from Utils import AMP_DTYPE + + model = torch.load( + root / "weights/23-36-37/model_best_bp2_serialize.pth", + map_location="cpu", + weights_only=False, + ) + finally: + os.chdir(previous) + model.args.max_disp = max_disparity + model.args.valid_iters = valid_iters + model.args.normalize = True + return model.cuda().eval(), InputPadder, AMP_DTYPE + + +def _image(path: str, *, height: int, width: int): + import numpy as np + from PIL import Image + + with Image.open(path) as image: + pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) + if pixels.shape != (height, width, 3): + raise ValueError(f"stereo image must be {height}x{width} RGB, got {pixels.shape}") + return pixels + + +def _inference( + model: Any, + input_padder: Any, + amp_dtype: Any, + left: Any, + right: Any, + *, + height: int, + width: int, + valid_iters: int, +): + import numpy as np + import torch + + left_tensor = torch.as_tensor(left).cuda().float()[None].permute(0, 3, 1, 2) + right_tensor = torch.as_tensor(right).cuda().float()[None].permute(0, 3, 1, 2) + padder = input_padder(left_tensor.shape, divis_by=32, force_square=False) + left_tensor, right_tensor = padder.pad(left_tensor, right_tensor) + with torch.inference_mode(), torch.amp.autocast("cuda", enabled=True, dtype=amp_dtype): + disparity = model.forward( + left_tensor, + right_tensor, + iters=valid_iters, + test_mode=True, + optimize_build_volume="pytorch1", + ) + values = padder.unpad(disparity.float()).cpu().numpy().reshape(height, width) + return np.clip(values, 0, None).astype(" tuple[int, int, int, int]: + height = int(request.get("height", 700)) + width = int(request.get("width", 700)) + max_disparity = int(request.get("max_disp", 192)) + valid_iters = int(request.get("valid_iters", 8)) + if (height, width, max_disparity, valid_iters) != (700, 700, 192, 8): + raise ValueError( + "Fast Foundation Stereo reference requires 700x700, max_disp=192, valid_iters=8" + ) + return height, width, max_disparity, valid_iters + + +def _summary(values: Any, artifact: Path) -> dict[str, Any]: + import numpy as np + + values.tofile(artifact) + return { + "height": int(values.shape[0]), + "width": int(values.shape[1]), + "element_count": int(values.size), + "finite_fraction": float(np.isfinite(values).mean()), + "nonnegative_fraction": float((values >= 0).mean()), + "disparity_artifact": str(artifact.resolve()), + } + + +def _run_accuracy(arguments: argparse.Namespace) -> int: + payload = json.loads(arguments.request.read_text(encoding="utf-8")) + request = payload.get("request", {}) + samples = payload.get("samples") + if not isinstance(request, Mapping) or not isinstance(samples, list) or not samples: + raise ValueError("Accuracy reference requires request and samples") + height, width, max_disparity, valid_iters = _options(request) + model, input_padder, amp_dtype = _load_model(max_disparity, valid_iters) + output_samples = [] + for index, sample in enumerate(samples): + if not isinstance(sample, Mapping): + raise ValueError(f"Accuracy sample {index} must be an object") + left = _image(str(sample["left_image_path"]), height=height, width=width) + right = _image(str(sample["right_image_path"]), height=height, width=width) + values = _inference( + model, + input_padder, + amp_dtype, + left, + right, + height=height, + width=width, + valid_iters=valid_iters, + ) + artifact = arguments.output.with_name( + f"{arguments.output.stem}.{index}.disparity.f32" + ) + output_samples.append( + {"sample_id": str(sample["sample_id"]), **_summary(values, artifact)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": output_samples}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +def _run_performance(arguments: argparse.Namespace) -> int: + import torch + + request = json.loads(arguments.request_json) + timing = json.loads(arguments.timing_contract_json) + height, width, max_disparity, valid_iters = _options(request) + model, input_padder, amp_dtype = _load_model(max_disparity, valid_iters) + left = _image(str(request["left_image_path"]), height=height, width=width) + right = _image(str(request["right_image_path"]), height=height, width=width) + + def invoke(): + return _inference( + model, + input_padder, + amp_dtype, + left, + right, + height=height, + width=width, + valid_iters=valid_iters, + ) + + result = None + for _ in range(arguments.warmup): + result = invoke() + samples_ms = [] + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter_ns() + result = invoke() + torch.cuda.synchronize() + samples_ms.append((time.perf_counter_ns() - started) / 1_000_000) + if result is None: + raise RuntimeError("reference produced no disparity output") + artifact = arguments.output.with_suffix(".disparity.f32") + arguments.output.parent.mkdir(parents=True, exist_ok=True) + summary = _summary(result, artifact) + value = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + "framework": f"torch-{torch.__version__}", + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": timing, + **timing, + "samples_ms": samples_ms, + "metrics": {"latency_ms": {"p50": statistics.median(samples_ms)}}, + "output_summary": summary, + } + arguments.output.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + for name in ( + "model", + "family", + "manifest", + "operation", + "selected-task", + "request-json", + "adapter-options-json", + "timing-contract-json", + "precision", + "mode", + "padding", + "case-name", + ): + parser.add_argument("--" + name, required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + return parser + + +def main() -> int: + if "--request" in sys.argv[1:]: + return _run_accuracy(_accuracy_parser().parse_args()) + return _run_performance(_performance_parser().parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/fast_foundation_stereo/tests/data/benchmark_stereo.json b/families/fast_foundation_stereo/tests/data/benchmark_stereo.json new file mode 100644 index 0000000000..622c4addf6 --- /dev/null +++ b/families/fast_foundation_stereo/tests/data/benchmark_stereo.json @@ -0,0 +1,9 @@ +{ + "requests": [ + { + "id": "office", + "left_image": "office_left.png", + "right_image": "office_right.png" + } + ] +} diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 0c0d43aa9f..eabb2c51a6 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -5,6 +5,7 @@ from __future__ import annotations +from array import array import csv from itertools import permutations import json @@ -32,7 +33,7 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, output = context.case_artifacts(case) output.mkdir(parents=True, exist_ok=True) definition = load_benchmark(context.repository, case) - dataset = resolve_dataset(definition, context) + dataset = resolve_dataset(definition, context, case.source) metric = definition.get("metric") metric_name = metric.get("name") if isinstance(metric, Mapping) else None if metric_name == "exact_token_ids": @@ -63,12 +64,228 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _ocr_text_parity(case, context, dataset, output) elif metric_name == "localization_text_parity": result = _localization_text_parity(case, context, dataset, output) + elif metric_name == "stereo_disparity_parity": + result = _stereo_disparity_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) return result +def _stereo_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("stereo dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"stereo dataset requires at least {sample_limit} requests") + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"stereo request {index} must be an object") + sample = {"sample_id": str(request.get("id") or f"sample-{index}")} + for source, target in ( + ("left_image", "left_image_path"), + ("right_image", "right_image_path"), + ): + relative = request.get(source) + if not isinstance(relative, str) or not relative: + raise QualificationError(f"stereo request {index} has no {source}") + image = (root / relative).resolve() + if not image.is_relative_to(root) or not image.is_file(): + raise QualificationError( + f"stereo request {index} {source} is unavailable: {image}" + ) + sample[target] = str(image) + selected.append(sample) + return selected + + +def _disparity_values(value: Mapping[str, Any], label: str) -> list[float]: + height = value.get("height") + width = value.get("width") + count = value.get("element_count", value.get("disparity_pixels")) + artifact = value.get("disparity_artifact") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or isinstance(count, bool) + or not isinstance(count, int) + or count != height * width + or not isinstance(artifact, str) + ): + raise QualificationError(f"{label} disparity output is incomplete") + try: + payload = Path(artifact).read_bytes() + except OSError as error: + raise QualificationError(f"{label} disparity artifact is unavailable") from error + if len(payload) != count * 4: + raise QualificationError(f"{label} disparity artifact has the wrong size") + values = array("f") + values.frombytes(payload) + if len(values) != count: + raise QualificationError(f"{label} disparity artifact has the wrong element count") + return list(values) + + +def _stereo_disparity_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _stereo_samples(dataset, sample_limit) + request = configured.get("request", {}) + reference = configured.get("reference", {}) + if not isinstance(request, Mapping) or not isinstance(reference, Mapping): + raise QualificationError("stereo Accuracy request and reference must be objects") + command = reference.get("command") + if not isinstance(command, str) or not command: + raise QualificationError("stereo Accuracy reference.command must be set") + runner = (case.source.parent / command).resolve() + family_root = case.source.parents[2].resolve() + if not runner.is_relative_to(family_root) or not runner.is_file(): + raise QualificationError(f"stereo reference runner is not family-owned: {runner}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp16")), + "request": dict(request), + "samples": selected, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"stereo Accuracy reference failed; see {output}") + try: + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + except (OSError, json.JSONDecodeError) as error: + raise QualificationError("stereo Accuracy reference returned unreadable output") from error + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("stereo Accuracy reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + **dict(request), + "left_image_path": sample["left_image_path"], + "right_image_path": sample["right_image_path"], + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs( + case, context, output, "disparity", candidate_requests + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("stereo Accuracy gate must be an object") + minimum_cosine = float(gate.get("min_cosine", 0.999)) + maximum_mean_error = float(gate.get("max_mean_abs_error", 0.5)) + maximum_bad_fraction = float(gate.get("max_bad_2px_fraction", 0.03)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= minimum_cosine <= 1.0 or not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("stereo cosine and pass-rate gates must be in [0, 1]") + if ( + not math.isfinite(maximum_mean_error) + or maximum_mean_error < 0.0 + or not 0.0 <= maximum_bad_fraction <= 1.0 + ): + raise QualificationError("stereo error gates are invalid") + + rows = [] + for sample, candidate_sample, reference_sample in zip( + selected, actual, expected, strict=True + ): + candidate_values = _disparity_values(candidate_sample, "candidate") + reference_values = _disparity_values(reference_sample, "reference") + if len(candidate_values) != len(reference_values): + raise QualificationError("candidate and reference disparity shapes differ") + candidate_valid = all(math.isfinite(value) and value >= 0.0 for value in candidate_values) + reference_valid = all(math.isfinite(value) and value >= 0.0 for value in reference_values) + dot = math.fsum( + left * right + for left, right in zip(candidate_values, reference_values, strict=True) + ) + candidate_norm = math.sqrt(math.fsum(value * value for value in candidate_values)) + reference_norm = math.sqrt(math.fsum(value * value for value in reference_values)) + cosine = ( + dot / (candidate_norm * reference_norm) + if candidate_norm and reference_norm + else 0.0 + ) + differences = [ + abs(left - right) + for left, right in zip(candidate_values, reference_values, strict=True) + ] + mean_error = math.fsum(differences) / len(differences) + bad_fraction = sum(value > 2.0 for value in differences) / len(differences) + passed = ( + candidate_valid + and reference_valid + and cosine >= minimum_cosine + and mean_error <= maximum_mean_error + and bad_fraction <= maximum_bad_fraction + ) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": passed, + "cosine": cosine, + "mean_abs_error": mean_error, + "bad_2px_fraction": bad_fraction, + } + ) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": pass_rate, + "min_cosine": min(row["cosine"] for row in rows), + "max_mean_abs_error": max(row["mean_abs_error"] for row in rows), + "max_bad_2px_fraction": max(row["bad_2px_fraction"] for row in rows), + }, + "gate": { + "min_cosine": minimum_cosine, + "max_mean_abs_error": maximum_mean_error, + "max_bad_2px_fraction": maximum_bad_fraction, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + def _text_generation_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml b/tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml new file mode 100644 index 0000000000..76b25771e1 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: stereo-office-pair + version: family-office-pair-v1 + sha256: ec5cac38fb583e821b7e19498d04eccd7bf22cb70718fa0d051090d6cdf1e893 + source: + mode: family + path: ../data/benchmark_stereo.json + license: Apache-2.0 +selection: + method: first +input: + adapter: stereo_image_paths +metric: + name: stereo_disparity_parity diff --git a/tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml b/tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/datasets.py b/tools/benchmark_qualification/datasets.py index 945579e405..37b9edd6bb 100644 --- a/tools/benchmark_qualification/datasets.py +++ b/tools/benchmark_qualification/datasets.py @@ -33,7 +33,11 @@ def receipt(self) -> dict[str, Any]: } -def resolve_dataset(definition: Mapping[str, Any], context: RuntimeContext) -> Dataset: +def resolve_dataset( + definition: Mapping[str, Any], + context: RuntimeContext, + profile: Path | None = None, +) -> Dataset: configured = definition.get("dataset") if not isinstance(configured, Mapping): raise QualificationError("Accuracy benchmark requires a dataset object") @@ -62,9 +66,21 @@ def resolve_dataset(definition: Mapping[str, Any], context: RuntimeContext) -> D raise QualificationError(f"downloadable dataset {dataset_id!r} requires dataset.sha256") path = _download_dataset(dataset_id, source, context.data_root) resolved_mode = "download" + elif mode == "family": + relative = source.get("path") + if profile is None: + raise QualificationError(f"family dataset {dataset_id!r} requires its model profile") + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise QualificationError("family dataset source.path must be relative") + family_root = profile.parents[2].resolve() + path = (profile.parent / relative).resolve() + if not path.is_relative_to(family_root): + raise QualificationError("family dataset source.path must stay inside its family") + resolved_mode = "family" else: raise QualificationError( - f"dataset {dataset_id!r} has unsupported source mode {mode!r}; use manual or download" + f"dataset {dataset_id!r} has unsupported source mode {mode!r}; " + "use manual, download, or family" ) if not path.is_file(): if mode == "manual": diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index e9c96fb4f7..b30730151a 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -3,6 +3,7 @@ from __future__ import annotations +from array import array import hashlib import json import os @@ -2106,3 +2107,169 @@ def download(_url, destination): assert resolved.path == tmp_path / "cache/public/public.jsonl" assert resolved.path.read_bytes() == payload assert resolve_dataset(definition, context).path == resolved.path + + +def test_family_dataset_stays_with_its_discovered_model_profile(tmp_path: Path) -> None: + profile = tmp_path / "families/stereo/tests/benchmark/stereo.yaml" + dataset = tmp_path / "families/stereo/tests/data/pairs.json" + profile.parent.mkdir(parents=True) + dataset.parent.mkdir(parents=True) + dataset.write_text('{"requests": []}\n', encoding="utf-8") + digest = hashlib.sha256(dataset.read_bytes()).hexdigest() + context = RuntimeContext( + repository=tmp_path, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "cache", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + definition = { + "dataset": { + "id": "family-pairs", + "sha256": digest, + "source": {"mode": "family", "path": "../data/pairs.json"}, + } + } + + resolved = resolve_dataset(definition, context, profile) + + assert resolved.path == dataset + assert resolved.receipt()["source_mode"] == "family" + definition["dataset"]["source"]["path"] = "../../../../outside.json" + with pytest.raises(QualificationError, match="inside its family"): + resolve_dataset(definition, context, profile) + + +def test_stereo_accuracy_compares_complete_disparity_artifacts( + tmp_path: Path, monkeypatch +) -> None: + profile = tmp_path / "families/stereo/tests/benchmark/stereo.yaml" + reference_script = profile.parent / "reference.py" + reference_script.parent.mkdir(parents=True) + reference_script.write_text("# family reference\n", encoding="utf-8") + data = tmp_path / "data" + data.mkdir() + for name in ("left.png", "right.png"): + (data / name).write_bytes(b"fixture") + dataset_path = data / "pairs.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "id": "pair", + "left_image": "left.png", + "right_image": "right.png", + } + ] + } + ), + encoding="utf-8", + ) + dataset = Dataset("stereo", dataset_path, "provided", "digest") + case = QualificationCase( + kind="accuracy", + model="stereo-model", + family="stereo", + name="parity", + benchmark="stereo_disparity_parity", + candidate={ + "family": "stereo", + "checkpoint": "example/stereo", + "task": "stereo_disparity", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "request": {"height": 1, "width": 2}, + "reference": {"command": "reference.py", "precision": "fp16"}, + "gate": { + "min_cosine": 0.99, + "max_mean_abs_error": 0.2, + "max_bad_2px_fraction": 0.0, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data-root", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + reference_artifact = tmp_path / "reference.f32" + reference_artifact.write_bytes(array("f", [1.0, 2.0]).tobytes()) + candidate_artifact = tmp_path / "candidate.f32" + candidate_artifact.write_bytes(array("f", [1.1, 2.1]).tobytes()) + + def run_reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "pair", + "height": 1, + "width": 2, + "element_count": 2, + "disparity_artifact": str(reference_artifact), + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "stereo_disparity_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", run_reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [ + { + "height": 1, + "width": 2, + "element_count": 2, + "disparity_artifact": str(candidate_artifact), + } + ], + tmp_path / "stereo.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["sample_pass_rate"] == 1.0 + assert result["metrics"]["max_mean_abs_error"] == pytest.approx(0.1) From 0af8dbcc0c352236ea56ffd1b8882dfd864c82dc Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 00:03:17 +0000 Subject: [PATCH 055/100] feat(benchmark): qualify MoGe geometry Signed-off-by: chaofengw --- .../trtmc_benchmark/artifact_metrics.py | 171 ++++++++++++++ .../tests/test_structured_output_contracts.py | 27 +++ .../moge/tests/benchmark/moge-2-vitl.yaml | 61 +++++ .../tests/benchmark/prepare_environment.py | 58 +++++ families/moge/tests/benchmark/reference.py | 217 ++++++++++++++++++ .../moge/tests/data/benchmark_geometry.json | 8 + families/moge/tests/reference_support.py | 112 +++++++++ tools/benchmark_qualification/accuracy.py | 69 ++++++ .../benchmarks/metric_geometry_parity.yaml | 19 ++ .../metric_geometry_performance.yaml | 15 ++ tools/perf_matrix.py | 21 ++ tools/tests/test_model_benchmark.py | 123 ++++++++++ 12 files changed, 901 insertions(+) create mode 100644 apps/benchmark/trtmc_benchmark/artifact_metrics.py create mode 100644 families/moge/tests/benchmark/moge-2-vitl.yaml create mode 100644 families/moge/tests/benchmark/prepare_environment.py create mode 100644 families/moge/tests/benchmark/reference.py create mode 100644 families/moge/tests/data/benchmark_geometry.json create mode 100644 families/moge/tests/reference_support.py create mode 100644 tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml diff --git a/apps/benchmark/trtmc_benchmark/artifact_metrics.py b/apps/benchmark/trtmc_benchmark/artifact_metrics.py new file mode 100644 index 0000000000..40e8c5928a --- /dev/null +++ b/apps/benchmark/trtmc_benchmark/artifact_metrics.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic numerical metrics over complete task output artifacts.""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any, Mapping + + +def metric_geometry_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare two complete metric-geometry outputs on their common valid grid.""" + import numpy as np + + actual = _metric_geometry(candidate, "candidate", np) + expected = _metric_geometry(reference, "reference", np) + actual_points, actual_depth, actual_mask, actual_intrinsics = actual + ref_points, ref_depth, ref_mask, ref_intrinsics = expected + if actual_depth.shape != ref_depth.shape: + raise ValueError("candidate and reference geometry grids differ") + actual_valid = actual_mask.astype(bool, copy=False) + ref_valid = ref_mask.astype(bool, copy=False) + common = actual_valid & ref_valid + union = actual_valid | ref_valid + if not np.any(common): + raise ValueError("candidate and reference geometry have no common valid pixels") + + actual_depth_valid = actual_depth[common].astype(np.float64) + ref_depth_valid = ref_depth[common].astype(np.float64) + depth_delta = actual_depth_valid - ref_depth_valid + actual_points_valid = actual_points[common].astype(np.float64) + ref_points_valid = ref_points[common].astype(np.float64) + point_delta = actual_points_valid - ref_points_valid + cosine_denominator = np.maximum( + np.linalg.norm(actual_points_valid, axis=-1) + * np.linalg.norm(ref_points_valid, axis=-1), + 1.0e-12, + ) + ref_intrinsics64 = ref_intrinsics.astype(np.float64) + intrinsics_delta = actual_intrinsics.astype(np.float64) - ref_intrinsics64 + nonzero = np.abs(ref_intrinsics64) > 1.0e-12 + metrics = { + "mask_iou": float(common.sum() / union.sum()), + "depth_absrel_mean": float( + np.mean(np.abs(depth_delta) / np.maximum(np.abs(ref_depth_valid), 1.0e-12)) + ), + "depth_rel_l2": float( + np.linalg.norm(depth_delta) + / max(float(np.linalg.norm(ref_depth_valid)), 1.0e-12) + ), + "points_rel_l2": float( + np.linalg.norm(point_delta) + / max(float(np.linalg.norm(ref_points_valid)), 1.0e-12) + ), + "points_cosine": float( + np.mean( + np.sum(actual_points_valid * ref_points_valid, axis=-1) + / cosine_denominator + ) + ), + "intrinsics_max_relative_error": float( + np.max(np.abs(intrinsics_delta[nonzero]) / np.abs(ref_intrinsics64[nonzero])) + ), + "point_depth_consistency": float( + np.max(np.abs(actual_points[..., 2][actual_valid] - actual_depth[actual_valid])) + ), + } + if not all(math.isfinite(value) for value in metrics.values()): + raise ValueError("metric-geometry comparison produced non-finite metrics") + return metrics + + +def metric_geometry_passes( + metrics: Mapping[str, float], thresholds: Mapping[str, Any] +) -> bool: + """Apply the public metric-geometry threshold vocabulary.""" + required = { + "mask_iou", + "depth_absrel_mean", + "depth_rel_l2", + "points_rel_l2", + "points_cosine", + "intrinsics_max_relative_error", + "point_depth_consistency", + } + if set(metrics) != required or not required.issubset(thresholds): + raise ValueError("metric-geometry thresholds are incomplete") + minimums = {"mask_iou", "points_cosine"} + for name in required: + threshold = thresholds[name] + if isinstance(threshold, bool) or not isinstance(threshold, (int, float)): + raise ValueError(f"metric-geometry threshold {name} must be numeric") + if not math.isfinite(float(threshold)): + raise ValueError(f"metric-geometry threshold {name} must be finite") + if name in minimums: + if metrics[name] < float(threshold): + return False + elif metrics[name] > float(threshold): + return False + return True + + +def _metric_geometry(summary: Mapping[str, Any], label: str, np): + height = _integer(summary.get("height"), f"{label} height", minimum=1) + width = _integer(summary.get("width"), f"{label} width", minimum=1) + pixels = height * width + if _integer(summary.get("geometry_images"), f"{label} geometry_images", minimum=1) != 1: + raise ValueError(f"{label} geometry must contain one image") + if _integer(summary.get("geometry_pixels"), f"{label} geometry_pixels", minimum=1) != pixels: + raise ValueError(f"{label} geometry pixel count differs from its grid") + if summary.get("point_shape") != [height, width, 3]: + raise ValueError(f"{label} point shape differs from [H,W,3]") + if ( + summary.get("units") != "meters" + or summary.get("camera_axes") != ["right", "down", "forward"] + or summary.get("intrinsics_coordinates") != "normalized_uv" + ): + raise ValueError(f"{label} geometry coordinate contract is invalid") + points = _float_artifact(summary, "points_artifact", (height, width, 3), label, np) + depth = _float_artifact(summary, "depth_artifact", (height, width), label, np) + mask = _mask_artifact(summary, pixels, (height, width), label, np) + intrinsics = np.asarray(summary.get("normalized_intrinsics"), dtype=np.float32) + if intrinsics.shape != (3, 3) or not np.isfinite(intrinsics).all(): + raise ValueError(f"{label} normalized intrinsics are invalid") + valid = mask.astype(bool, copy=False) + if not np.any(valid): + raise ValueError(f"{label} geometry has no valid pixels") + if not np.isfinite(points[valid]).all(): + raise ValueError(f"{label} valid points are not finite") + if not np.isfinite(depth[valid]).all() or np.any(depth[valid] <= 0.0): + raise ValueError(f"{label} valid depth is not finite and positive") + declared_valid = _integer(summary.get("valid_pixels"), f"{label} valid_pixels") + if declared_valid != int(valid.sum()): + raise ValueError(f"{label} valid pixel count differs from its mask") + return points, depth, mask, intrinsics + + +def _float_artifact(summary, name, shape, label, np): + path = _artifact_path(summary.get(name), f"{label} {name}") + values = np.fromfile(path, dtype=" Path: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} is missing") + path = Path(value) + if not path.is_file(): + raise ValueError(f"{label} is unavailable") + return path + + +def _integer(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{label} must be an integer >= {minimum}") + return value diff --git a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py b/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py index 6428902993..c375e6d87e 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py @@ -283,6 +283,33 @@ def test_geometry_shape_does_not_filter_invalid_infinity_or_add_depth_accuracy_r assert struct.unpack("<2f", originals["depth_artifact"])[1] == float("inf") +def test_geometry_parity_compares_complete_numerical_artifacts(tmp_path): + left = _geometry(tmp_path / "candidate") + right = _geometry(tmp_path / "reference") + entry = _entry("geometry") + entry.spec["baseline"].update( + output_contract="metric-geometry-parity", + mask_iou=0.999, + depth_absrel_mean=0.005, + depth_rel_l2=0.02, + points_rel_l2=0.02, + points_cosine=0.99999, + intrinsics_max_relative_error=0.002, + point_depth_consistency=0.00001, + ) + + matched, reason, evidence = perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + ) + + assert matched is True and reason == "" + assert evidence["numerical_parity_checked"] is True + Path(right["depth_artifact"]).write_bytes(struct.pack("<2f", 6, float("inf"))) + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + + def test_optional_reference_calibration_artifact_is_not_silently_ignored(tmp_path): left = _geometry(tmp_path / "candidate") right = _geometry(tmp_path / "reference") diff --git a/families/moge/tests/benchmark/moge-2-vitl.yaml b/families/moge/tests/benchmark/moge-2-vitl.yaml new file mode 100644 index 0000000000..f38c24b510 --- /dev/null +++ b/families/moge/tests/benchmark/moge-2-vitl.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: moge-2-vitl +candidate: + family: moge + checkpoint: Ruicheng/moge-2-vitl + revision: 39c4d5e957afe587e04eec59dc2bcc3be5ecd968 + task: monocular_geometry + precision: fp32 + bundle: moge-2-vitl.bundle + build: + tensor_parallel_size: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: metric-geometry-parity + benchmark: metric_geometry_parity + samples: 1 + request: &geometry-request + num_tokens: 1800 + reference: + command: reference.py + precision: fp32 + gate: + mask_iou: 0.999 + depth_absrel_mean: 0.005 + depth_rel_l2: 0.02 + points_rel_l2: 0.02 + points_cosine: 0.99999 + intrinsics_max_relative_error: 0.002 + point_depth_consistency: 0.00001 + min_sample_pass_rate: 1.0 + +performance: + - name: geometry + benchmark: metric_geometry_performance + operation: geometry + request: + <<: *geometry-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: official_pytorch + precision: fp32 + output_contract: metric-geometry-parity + mask_iou: 0.999 + depth_absrel_mean: 0.005 + depth_rel_l2: 0.02 + points_rel_l2: 0.02 + points_cosine: 0.99999 + intrinsics_max_relative_error: 0.002 + point_depth_consistency: 0.00001 diff --git a/families/moge/tests/benchmark/prepare_environment.py b/families/moge/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4c525076e4 --- /dev/null +++ b/families/moge/tests/benchmark/prepare_environment.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned official MoGe source inside this family's environment.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +SOURCE_REPOSITORY = "https://github.com/microsoft/MoGe.git" +SOURCE_REVISION = "74fbce054ebed49800de42d0ad0e83495065719a" + + +def main() -> int: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "MoGe" + required = destination / "moge/model/v2.py" + if required.is_file(): + return 0 + if destination.exists(): + raise RuntimeError(f"incomplete MoGe reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="MoGe.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", SOURCE_REPOSITORY], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(temporary), + "fetch", + "--quiet", + "--depth=1", + "origin", + SOURCE_REVISION, + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/moge/tests/benchmark/reference.py b/families/moge/tests/benchmark/reference.py new file mode 100644 index 0000000000..2e719b5180 --- /dev/null +++ b/families/moge/tests/benchmark/reference.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned official Accuracy and Performance reference for MoGe-2.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import statistics +import sys +import time +from typing import Any, Mapping + +import numpy as np + +from families.moge.tests.reference_support import OfficialReference, read_image + + +def _source_root() -> Path: + root = Path(sys.prefix) / "trtmc-reference/MoGe" + if not (root / "moge/model/v2.py").is_file(): + raise RuntimeError(f"MoGe reference is incomplete: {root}") + return root + + +def _checkpoint(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import hf_hub_download + + return Path( + hf_hub_download( + repo_id=model, + revision=revision, + filename="model.pt", + local_files_only=local_files_only, + ) + ) + + +def _options(request: Mapping[str, Any]) -> tuple[int, float | None]: + unknown = sorted(set(request) - {"image_path", "num_tokens", "fov_x"}) + if unknown: + raise ValueError("unsupported MoGe request fields: " + ", ".join(unknown)) + num_tokens = request.get("num_tokens") + if isinstance(num_tokens, bool) or not isinstance(num_tokens, int) or num_tokens != 1800: + raise ValueError("MoGe reference requires num_tokens=1800") + fov_x = request.get("fov_x") + if fov_x is not None: + fov_x = float(fov_x) + if not 0.0 < fov_x < 180.0: + raise ValueError("MoGe fov_x must be between zero and 180 degrees") + return num_tokens, fov_x + + +def _write_summary(arrays: Mapping[str, Any], prefix: Path) -> dict[str, Any]: + points = np.asarray(arrays["points"], dtype=" int: + payload = json.loads(arguments.request.read_text(encoding="utf-8")) + request = payload.get("request", {}) + samples = payload.get("samples") + if not isinstance(request, Mapping) or not isinstance(samples, list) or not samples: + raise ValueError("Accuracy reference requires request and samples") + num_tokens, fov_x = _options(request) + model = OfficialReference( + _source_root(), + _checkpoint( + str(payload["model"]), + payload.get("revision"), + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + ), + ) + output_samples = [] + for index, sample in enumerate(samples): + if not isinstance(sample, Mapping): + raise ValueError(f"Accuracy sample {index} must be an object") + arrays = model.infer( + read_image(Path(str(sample["image_path"]))), + num_tokens=num_tokens, + fov_x=fov_x, + ) + summary = _write_summary( + arrays, arguments.output.with_name(f"{arguments.output.stem}.{index}.geometry") + ) + output_samples.append({"sample_id": str(sample["sample_id"]), **summary}) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": output_samples}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +def _run_performance(arguments: argparse.Namespace) -> int: + import torch + + request = json.loads(arguments.request_json) + timing = json.loads(arguments.timing_contract_json) + num_tokens, fov_x = _options(request) + reference = OfficialReference( + _source_root(), + _checkpoint(arguments.model, arguments.revision, arguments.local_files_only), + ) + pixels = read_image(Path(str(request["image_path"]))) + + def invoke(): + return reference.infer(pixels, num_tokens=num_tokens, fov_x=fov_x) + + arrays = None + for _ in range(arguments.warmup): + arrays = invoke() + samples_ms = [] + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter_ns() + arrays = invoke() + torch.cuda.synchronize() + samples_ms.append((time.perf_counter_ns() - started) / 1_000_000) + if arrays is None: + raise RuntimeError("MoGe reference produced no geometry") + summary = _write_summary(arrays, arguments.output.with_suffix(".geometry")) + value = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + "framework": f"torch-{torch.__version__}", + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": timing, + **timing, + "samples_ms": samples_ms, + "metrics": {"latency_ms": {"p50": statistics.median(samples_ms)}}, + "output_summary": summary, + } + arguments.output.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + for name in ( + "model", + "family", + "manifest", + "operation", + "selected-task", + "request-json", + "adapter-options-json", + "timing-contract-json", + "precision", + "mode", + "padding", + "case-name", + ): + parser.add_argument("--" + name, required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + return parser + + +def main() -> int: + if "--request" in sys.argv[1:]: + return _run_accuracy(_accuracy_parser().parse_args()) + return _run_performance(_performance_parser().parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/moge/tests/data/benchmark_geometry.json b/families/moge/tests/data/benchmark_geometry.json new file mode 100644 index 0000000000..67df60ae86 --- /dev/null +++ b/families/moge/tests/data/benchmark_geometry.json @@ -0,0 +1,8 @@ +{ + "requests": [ + { + "id": "moge-test-image", + "image": "test_img.jpeg" + } + ] +} diff --git a/families/moge/tests/reference_support.py b/families/moge/tests/reference_support.py new file mode 100644 index 0000000000..e3e2b04cc6 --- /dev/null +++ b/families/moge/tests/reference_support.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned pinned MoGe-2 reference loading and inference.""" + +from __future__ import annotations + +import contextlib +from pathlib import Path +import sys +import types + +import numpy as np + + +def _math_sdpa(torch): + try: + from torch.nn.attention import SDPBackend, sdpa_kernel + + return sdpa_kernel([SDPBackend.MATH]) + except (ImportError, AttributeError): + return contextlib.nullcontext() + + +def _install_utils3d_shim(torch) -> None: + def intrinsics_from_focal_center(fx, fy, cx, cy): + fx, fy, cx, cy = torch.broadcast_tensors( + torch.as_tensor(fx), + torch.as_tensor(fy), + torch.as_tensor(cx, device=torch.as_tensor(fx).device), + torch.as_tensor(cy, device=torch.as_tensor(fx).device), + ) + matrix = torch.zeros((*fx.shape, 3, 3), dtype=fx.dtype, device=fx.device) + matrix[..., 0, 0] = fx + matrix[..., 1, 1] = fy + matrix[..., 0, 2] = cx + matrix[..., 1, 2] = cy + matrix[..., 2, 2] = 1.0 + return matrix + + def depth_map_to_point_map(depth, *, intrinsics): + height, width = depth.shape[-2:] + u = (torch.arange(width, dtype=depth.dtype, device=depth.device) + 0.5) / width + v = (torch.arange(height, dtype=depth.dtype, device=depth.device) + 0.5) / height + u, v = torch.meshgrid(u, v, indexing="xy") + while u.ndim < depth.ndim: + u = u.unsqueeze(0) + v = v.unsqueeze(0) + x = (u - intrinsics[..., 0, 2, None, None]) / intrinsics[..., 0, 0, None, None] + y = (v - intrinsics[..., 1, 2, None, None]) / intrinsics[..., 1, 1, None, None] + return torch.stack((x * depth, y * depth, depth), dim=-1) + + shim = types.ModuleType("utils3d_moge") + shim.pt = types.SimpleNamespace( + intrinsics_from_focal_center=intrinsics_from_focal_center, + depth_map_to_point_map=depth_map_to_point_map, + ) + sys.modules["utils3d_moge"] = shim + sys.modules.setdefault("cv2", types.ModuleType("cv2")) + + +class OfficialReference: + def __init__(self, source_root: Path, checkpoint: Path): + import torch + + _install_utils3d_shim(torch) + source_root = source_root.resolve() + sys.path.insert(0, str(source_root)) + from moge.model.v2 import MoGeModel + + state = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if set(state) != {"model_config", "model"}: + raise ValueError("MoGe checkpoint has an unexpected top-level contract") + model = MoGeModel(**state["model_config"]) + missing, unexpected = model.load_state_dict(state["model"], strict=False) + if missing or unexpected: + raise ValueError(f"MoGe state mismatch: missing={missing}, unexpected={unexpected}") + model.onnx_compatible_mode = True + self.torch = torch + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model = model.eval().float().to(self.device) + if torch.cuda.is_available(): + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + def infer(self, pixels: np.ndarray, *, num_tokens: int, fov_x: float | None = None): + if num_tokens != 1800: + raise ValueError("the qualified MoGe reference requires num_tokens=1800") + tensor = self.torch.from_numpy(pixels).permute(2, 0, 1).to(self.device) + options = { + "num_tokens": num_tokens, + "use_fp16": False, + "force_projection": True, + "apply_mask": True, + } + if fov_x is not None: + options["fov_x"] = fov_x + with _math_sdpa(self.torch): + output = self.model.infer(tensor, **options) + required = {"points", "depth", "intrinsics", "mask"} + if set(output) != required: + raise ValueError( + f"MoGe reference returned {sorted(output)}, expected {sorted(required)}" + ) + return {name: tensor.detach().cpu().numpy() for name, tensor in output.items()} + + +def read_image(path: Path) -> np.ndarray: + from PIL import Image + + with Image.open(path) as image: + return np.asarray(image.convert("RGB"), dtype=np.float32).copy() / 255.0 diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index eabb2c51a6..302af03696 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -15,6 +15,8 @@ from pathlib import Path from typing import Any, Mapping, Sequence +from trtmc_benchmark.artifact_metrics import metric_geometry_metrics, metric_geometry_passes + from .catalog import QualificationCase, QualificationError, load_benchmark from .datasets import Dataset, resolve_dataset from .runtime import ( @@ -66,12 +68,79 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _localization_text_parity(case, context, dataset, output) elif metric_name == "stereo_disparity_parity": result = _stereo_disparity_parity(case, context, dataset, output) + elif metric_name == "metric_geometry_parity": + result = _metric_geometry_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) return result +def _metric_geometry_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "metric-geometry") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("metric-geometry request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "geometry", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("metric-geometry gate must be an object") + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("metric-geometry min_sample_pass_rate must be in [0, 1]") + thresholds = {key: value for key, value in gate.items() if key != "min_sample_pass_rate"} + rows = [] + for sample, candidate_sample, reference_sample in zip( + selected, actual, expected, strict=True + ): + try: + metrics = metric_geometry_metrics(candidate_sample, reference_sample) + passed = metric_geometry_passes(metrics, thresholds) + except (OSError, ValueError) as error: + raise QualificationError( + f"metric-geometry sample {sample['sample_id']} is invalid: {error}" + ) from error + rows.append({"sample_id": sample["sample_id"], "passed": passed, **metrics}) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + metric_names = tuple(thresholds) + aggregate = { + name: ( + min(float(row[name]) for row in rows) + if name in {"mask_iou", "points_cosine"} + else max(float(row[name]) for row in rows) + ) + for name in metric_names + } + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": {"samples": len(rows), "sample_pass_rate": pass_rate, **aggregate}, + "gate": dict(gate), + "samples": rows, + } + + def _stereo_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: try: payload = json.loads(dataset.path.read_text(encoding="utf-8")) diff --git a/tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml b/tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml new file mode 100644 index 0000000000..d901770179 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: metric-geometry-test-image + version: family-test-image-v1 + sha256: b600ba0a7d131e831dcb3ff4140759b8c3d6bd319345cf5f3756df8c8f1bf5c2 + source: + mode: family + path: ../data/benchmark_geometry.json + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: metric_geometry_parity diff --git a/tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml b/tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index e47c26e0a6..2468efff73 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -40,6 +40,10 @@ from apps.benchmark.performance.baselines.timing_contracts import timing_contract # noqa: E402 from apps.benchmark.performance.baselines.hf_transformers import flatten_config # noqa: E402 +from trtmc_benchmark.artifact_metrics import ( # noqa: E402 + metric_geometry_metrics, + metric_geometry_passes, +) from trtmc_benchmark.catalog import ManifestCatalog, resolve_case, selected_task_for_case # noqa: E402 from trtmc_benchmark.task_adapters import default_operation # noqa: E402 from trtmc_benchmark.types import BenchmarkError # noqa: E402 @@ -81,6 +85,7 @@ "localization", "media-shape", "metric-geometry-shape", + "metric-geometry-parity", "molecular-structure-shape", "normalized-text", "ocr-text", @@ -1360,6 +1365,22 @@ def _output_contract( ) if contract == "localization": return _localization_contract(entry, left, right) + if contract == "metric-geometry-parity": + try: + metrics = metric_geometry_metrics(left, right) + passed = metric_geometry_passes(metrics, entry.spec["baseline"]) + except (OSError, ValueError) as error: + return ( + False, + f"metric-geometry parity could not be evaluated: {error}", + {"contract": contract, "numerical_parity_checked": True}, + ) + evidence = { + "contract": contract, + "numerical_parity_checked": True, + **metrics, + } + return passed, "metric-geometry parity is outside the contract" if not passed else "", evidence if contract in {"metric-geometry-shape", "molecular-structure-shape", "pose-refinement-shape"}: signature = { "metric-geometry-shape": _metric_geometry_signature, diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index b30730151a..5402f2e235 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -2273,3 +2273,126 @@ def run_reference(command, *_args, **_kwargs): assert result["status"] == "passed" assert result["metrics"]["sample_pass_rate"] == 1.0 assert result["metrics"]["max_mean_abs_error"] == pytest.approx(0.1) + + +def test_metric_geometry_accuracy_compares_complete_task_artifacts( + tmp_path: Path, monkeypatch +) -> None: + profile = tmp_path / "families/moge/tests/benchmark/moge.yaml" + profile.parent.mkdir(parents=True) + image = tmp_path / "data/image.jpeg" + image.parent.mkdir() + image.write_bytes(b"fixture") + dataset_path = image.parent / "dataset.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), + encoding="utf-8", + ) + dataset = Dataset("geometry", dataset_path, "provided", "digest") + + def geometry(root: Path) -> dict[str, object]: + root.mkdir() + (root / "points.f32").write_bytes( + array("f", [1.0, 2.0, 3.0, float("inf"), float("inf"), float("inf")]).tobytes() + ) + (root / "depth.f32").write_bytes(array("f", [3.0, float("inf")]).tobytes()) + (root / "mask.u8").write_bytes(bytes((1, 0))) + return { + "geometry_images": 1, + "geometry_pixels": 2, + "height": 1, + "width": 2, + "point_shape": [1, 2, 3], + "valid_pixels": 1, + "normalized_intrinsics": [ + [1.0, 0.0, 0.5], + [0.0, 1.0, 0.5], + [0.0, 0.0, 1.0], + ], + "units": "meters", + "camera_axes": ["right", "down", "forward"], + "intrinsics_coordinates": "normalized_uv", + "points_artifact": str(root / "points.f32"), + "depth_artifact": str(root / "depth.f32"), + "valid_mask_artifact": str(root / "mask.u8"), + } + + candidate = geometry(tmp_path / "candidate") + reference = geometry(tmp_path / "reference") + thresholds = { + "mask_iou": 0.999, + "depth_absrel_mean": 0.005, + "depth_rel_l2": 0.02, + "points_rel_l2": 0.02, + "points_cosine": 0.99999, + "intrinsics_max_relative_error": 0.002, + "point_depth_consistency": 0.00001, + "min_sample_pass_rate": 1.0, + } + case = QualificationCase( + kind="accuracy", + model="moge", + family="moge", + name="geometry", + benchmark="metric_geometry_parity", + candidate={ + "family": "moge", + "checkpoint": "example/moge", + "task": "monocular_geometry", + "precision": "fp32", + "build": {}, + }, + values={ + "samples": 1, + "request": {"num_tokens": 1800}, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": thresholds, + }, + source=profile, + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data-root", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "metric_geometry_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "_family_image_reference", lambda *_args: [reference] + ) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ([candidate], tmp_path / "moge.bundle"), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["samples"] == 1 + assert result["metrics"]["sample_pass_rate"] == 1.0 + assert result["metrics"]["mask_iou"] == 1.0 + assert result["metrics"]["points_cosine"] == pytest.approx(1.0) + for name in ( + "depth_absrel_mean", + "depth_rel_l2", + "points_rel_l2", + "intrinsics_max_relative_error", + "point_depth_consistency", + ): + assert result["metrics"][name] == 0.0 From 17c94b0e607d54a04341eb0fe7f239b761420c23 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 00:26:07 +0000 Subject: [PATCH 056/100] feat(benchmark): qualify LeRobot ACT control Add family-owned Accuracy and Performance coverage for the pinned LeRobot ACT checkpoint and recorded observation. Compare every action value against the official PyTorch reference through the public control operation, and register the accumulated generic benchmark definitions in the architecture closed set. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 1 + .../trtmc_benchmark/artifact_metrics.py | 64 ++++++ .../tests/test_structured_output_contracts.py | 81 +++++++- .../act-aloha-sim-transfer-cube.yaml | 51 +++++ .../tests/benchmark/prepare_environment.py | 58 ++++++ .../lerobot_act/tests/benchmark/reference.py | 183 ++++++++++++++++++ .../tests/data/benchmark_robot_control.json | 9 + tools/benchmark_qualification/accuracy.py | 102 +++++++++- .../benchmarks/robot_action_parity.yaml | 19 ++ .../benchmarks/robot_action_performance.yaml | 15 ++ tools/perf_matrix.py | 20 ++ tools/tests/test_architecture.py | 14 +- tools/tests/test_model_benchmark.py | 102 ++++++++++ 13 files changed, 713 insertions(+), 6 deletions(-) create mode 100644 families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml create mode 100644 families/lerobot_act/tests/benchmark/prepare_environment.py create mode 100644 families/lerobot_act/tests/benchmark/reference.py create mode 100644 families/lerobot_act/tests/data/benchmark_robot_control.json create mode 100644 tools/benchmark_qualification/benchmarks/robot_action_parity.yaml create mode 100644 tools/benchmark_qualification/benchmarks/robot_action_performance.yaml diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 510ebf0d8c..27bda34121 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -686,6 +686,7 @@ Json run_control(trtmc::ITask& task, const Json& request, const Timing& timing) return Json{{"action_steps", result.num_actions}, {"action_dim", result.action_dim}, {"action_values", result.actions.size()}, + {"actions", result.actions}, {"within_training_bounds", result.within_training_bounds}, {"inference_ms", result.inference_ms}}; }); diff --git a/apps/benchmark/trtmc_benchmark/artifact_metrics.py b/apps/benchmark/trtmc_benchmark/artifact_metrics.py index 40e8c5928a..a48d290cac 100644 --- a/apps/benchmark/trtmc_benchmark/artifact_metrics.py +++ b/apps/benchmark/trtmc_benchmark/artifact_metrics.py @@ -104,6 +104,70 @@ def metric_geometry_passes( return True +def robot_action_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare every value in two robot action chunks.""" + actual = _robot_actions(candidate, "candidate") + expected = _robot_actions(reference, "reference") + if len(actual) != len(expected): + raise ValueError("candidate and reference action chunks differ in size") + deltas = [float(left) - float(right) for left, right in zip(actual, expected, strict=True)] + metrics = { + "action_max_abs_error": max(abs(value) for value in deltas), + "action_mean_abs_error": math.fsum(abs(value) for value in deltas) / len(deltas), + "action_rmse": math.sqrt(math.fsum(value * value for value in deltas) / len(deltas)), + } + if not all(math.isfinite(value) for value in metrics.values()): + raise ValueError("robot action comparison produced non-finite metrics") + return metrics + + +def robot_action_passes( + metrics: Mapping[str, float], thresholds: Mapping[str, Any] +) -> bool: + """Apply the public robot-action threshold vocabulary.""" + required = { + "action_max_abs_error", + "action_mean_abs_error", + "action_rmse", + } + if set(metrics) != required or not required.issubset(thresholds): + raise ValueError("robot action thresholds are incomplete") + for name in required: + threshold = thresholds[name] + if isinstance(threshold, bool) or not isinstance(threshold, (int, float)): + raise ValueError(f"robot action threshold {name} must be numeric") + if not math.isfinite(float(threshold)) or float(threshold) < 0.0: + raise ValueError(f"robot action threshold {name} must be finite and non-negative") + if metrics[name] > float(threshold): + return False + return True + + +def _robot_actions(summary: Mapping[str, Any], label: str) -> list[float]: + steps = _integer(summary.get("action_steps"), f"{label} action_steps", minimum=1) + dimension = _integer(summary.get("action_dim"), f"{label} action_dim", minimum=1) + count = _integer(summary.get("action_values"), f"{label} action_values", minimum=1) + values = summary.get("actions") + if count != steps * dimension: + raise ValueError(f"{label} action shape does not match its value count") + if not isinstance(values, list) or len(values) != count: + raise ValueError(f"{label} action values are missing or incomplete") + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + for value in values + ): + raise ValueError(f"{label} action values must be finite numbers") + if label == "candidate" and summary.get("within_training_bounds") is not True: + raise ValueError("candidate action values are outside the training bounds") + if label == "reference" and summary.get("finite") is not True: + raise ValueError("reference action values are not declared finite") + return [float(value) for value in values] + + def _metric_geometry(summary: Mapping[str, Any], label: str, np): height = _integer(summary.get("height"), f"{label} height", minimum=1) width = _integer(summary.get("width"), f"{label} width", minimum=1) diff --git a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py b/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py index c375e6d87e..d536b2ab6b 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py @@ -39,6 +39,25 @@ def _entry(operation): ) +def _control_entry(): + return perf.ResolvedEntry( + spec={"id": "robot-action-fixture", "operation": "control", "baseline": {}}, + model=SimpleNamespace(task="robot_control"), + case=SimpleNamespace( + selected_task=None, + request={}, + measurement=SimpleNamespace(asset_loading_included=False), + ), + manifest={}, + reference_precision="fp32", + baseline_timing={ + "timing_scope": "task-pipeline-call-wall", + "input_preparation_included": True, + "asset_loading_included": False, + }, + ) + + def _geometry(root: Path): root.mkdir() # Two HWC pixels: a valid metric point and an invalid +Inf sentinel. @@ -109,7 +128,22 @@ def _poses(_root: Path): } -FIXTURES = {"geometry": _geometry, "predict_structure": _structure, "refine_pose": _poses} +def _actions(_root: Path): + return { + "action_steps": 2, + "action_dim": 2, + "action_values": 4, + "actions": [0.1, -0.2, 0.3, -0.4], + "finite": True, + "within_training_bounds": True, + } + + +FIXTURES = { + "geometry": _geometry, + "predict_structure": _structure, + "refine_pose": _poses, +} def _compare(operation, left, right): @@ -310,6 +344,51 @@ def test_geometry_parity_compares_complete_numerical_artifacts(tmp_path): )[0] is False +def test_robot_action_parity_compares_every_action_value(tmp_path): + left = _actions(tmp_path / "candidate") + right = _actions(tmp_path / "reference") + entry = _control_entry() + entry.spec["baseline"].update( + output_contract="robot-action-parity", + action_max_abs_error=0.00005, + action_mean_abs_error=0.000005, + action_rmse=0.00001, + ) + + matched, reason, evidence = perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + ) + + assert matched is True and reason == "" + assert evidence["numerical_parity_checked"] is True + right["actions"][2] += 0.01 + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + + +def test_robot_action_parity_rejects_missing_or_out_of_bounds_values(tmp_path): + left = _actions(tmp_path / "candidate") + right = _actions(tmp_path / "reference") + entry = _control_entry() + entry.spec["baseline"].update( + output_contract="robot-action-parity", + action_max_abs_error=0.00005, + action_mean_abs_error=0.000005, + action_rmse=0.00001, + ) + + left["within_training_bounds"] = False + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + left["within_training_bounds"] = True + right["actions"].pop() + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + + def test_optional_reference_calibration_artifact_is_not_silently_ignored(tmp_path): left = _geometry(tmp_path / "candidate") right = _geometry(tmp_path / "reference") diff --git a/families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml b/families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml new file mode 100644 index 0000000000..791a7a8d65 --- /dev/null +++ b/families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: act-aloha-sim-transfer-cube +candidate: + family: lerobot_act + checkpoint: lerobot/act_aloha_sim_transfer_cube_human + revision: ba73b2766f1371cdc133ca4efb97eb090d744625 + task: robot_control + precision: fp32 + bundle: act-aloha-sim-transfer-cube.bundle + build: + tensor_parallel_size: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: recorded-action-parity + benchmark: robot_action_parity + samples: 1 + reference: + command: reference.py + precision: fp32 + gate: + action_max_abs_error: 0.00005 + action_mean_abs_error: 0.000005 + action_rmse: 0.00001 + min_sample_pass_rate: 1.0 + +performance: + - name: control + benchmark: robot_action_performance + operation: control + request: + image_path: ../data/recorded_observation/observation.images.top.png + state_path: ../data/recorded_observation/observation.state.f32 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: official_lerobot + precision: fp32 + output_contract: robot-action-parity + action_max_abs_error: 0.00005 + action_mean_abs_error: 0.000005 + action_rmse: 0.00001 diff --git a/families/lerobot_act/tests/benchmark/prepare_environment.py b/families/lerobot_act/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4874c92184 --- /dev/null +++ b/families/lerobot_act/tests/benchmark/prepare_environment.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned LeRobot source inside this family's environment.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +SOURCE_REPOSITORY = "https://github.com/huggingface/lerobot.git" +SOURCE_REVISION = "3c0a209f9fac4d2a57617e686a7f2a2309144ba2" + + +def main() -> int: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "lerobot" + required = destination / "lerobot/common/policies/act/modeling_act.py" + if required.is_file(): + return 0 + if destination.exists(): + raise RuntimeError(f"incomplete LeRobot reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="lerobot.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", SOURCE_REPOSITORY], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(temporary), + "fetch", + "--quiet", + "--depth=1", + "origin", + SOURCE_REVISION, + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/lerobot_act/tests/benchmark/reference.py b/families/lerobot_act/tests/benchmark/reference.py new file mode 100644 index 0000000000..567b48ab2e --- /dev/null +++ b/families/lerobot_act/tests/benchmark/reference.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned official Accuracy and Performance reference for LeRobot ACT.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import statistics +import sys +import time +from typing import Any, Mapping + +import numpy as np + +from families.lerobot_act.tests.official_reference import ( + load_observation, + load_policy, + predict_actions, +) + + +def _source_root() -> Path: + root = Path(sys.prefix) / "trtmc-reference/lerobot" + if not (root / "lerobot/common/policies/act/modeling_act.py").is_file(): + raise RuntimeError(f"LeRobot reference is incomplete: {root}") + return root + + +def _checkpoint(model: str, revision: str | None, local_files_only: bool) -> Path: + path = Path(model).expanduser() + if path.is_dir(): + return path.resolve() + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=local_files_only, + ) + ) + + +def _summary(actions: np.ndarray) -> dict[str, Any]: + return { + "action_steps": int(actions.shape[0]), + "action_dim": int(actions.shape[1]), + "action_values": int(actions.size), + "actions": actions.reshape(-1).tolist(), + "finite": bool(np.isfinite(actions).all()), + } + + +def _run_accuracy(arguments: argparse.Namespace) -> int: + payload = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = payload.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("Accuracy reference requires samples") + checkpoint = _checkpoint( + str(payload["model"]), + payload.get("revision"), + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + ) + torch, policy, config, device = load_policy(_source_root(), checkpoint) + output_samples = [] + for index, sample in enumerate(samples): + if not isinstance(sample, Mapping): + raise ValueError(f"Accuracy sample {index} must be an object") + pixels, state = load_observation( + Path(str(sample["image_path"])), Path(str(sample["state_path"])) + ) + actions = predict_actions(torch, policy, config, device, pixels, state) + output_samples.append( + {"sample_id": str(sample["sample_id"]), **_summary(actions)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": output_samples}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +def _run_performance(arguments: argparse.Namespace) -> int: + import torch + + request = json.loads(arguments.request_json) + timing = json.loads(arguments.timing_contract_json) + checkpoint = _checkpoint(arguments.model, arguments.revision, arguments.local_files_only) + _, policy, config, device = load_policy(_source_root(), checkpoint) + pixels, state = load_observation( + Path(str(request["image_path"])), Path(str(request["state_path"])) + ) + + def invoke(): + return predict_actions(torch, policy, config, device, pixels, state) + + actions = None + for _ in range(arguments.warmup): + actions = invoke() + samples_ms = [] + for _ in range(arguments.iterations): + if torch.cuda.is_available(): + torch.cuda.synchronize() + started = time.perf_counter_ns() + actions = invoke() + if torch.cuda.is_available(): + torch.cuda.synchronize() + samples_ms.append((time.perf_counter_ns() - started) / 1_000_000) + if actions is None: + raise RuntimeError("LeRobot reference produced no actions") + value = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + "framework": f"lerobot-torch-{torch.__version__}", + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": timing, + **timing, + "samples_ms": samples_ms, + "metrics": {"latency_ms": {"p50": statistics.median(samples_ms)}}, + "output_summary": _summary(actions), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + for name in ( + "model", + "family", + "manifest", + "operation", + "selected-task", + "request-json", + "adapter-options-json", + "timing-contract-json", + "precision", + "mode", + "padding", + "case-name", + ): + parser.add_argument("--" + name, required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + return parser + + +def main() -> int: + if "--request" in sys.argv[1:]: + return _run_accuracy(_accuracy_parser().parse_args()) + return _run_performance(_performance_parser().parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/lerobot_act/tests/data/benchmark_robot_control.json b/families/lerobot_act/tests/data/benchmark_robot_control.json new file mode 100644 index 0000000000..3ece00d678 --- /dev/null +++ b/families/lerobot_act/tests/data/benchmark_robot_control.json @@ -0,0 +1,9 @@ +{ + "requests": [ + { + "id": "aloha-sim-transfer-cube-episode-0-frame-0", + "image": "recorded_observation/observation.images.top.png", + "state": "recorded_observation/observation.state.f32" + } + ] +} diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 302af03696..4aff731a2e 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -15,7 +15,12 @@ from pathlib import Path from typing import Any, Mapping, Sequence -from trtmc_benchmark.artifact_metrics import metric_geometry_metrics, metric_geometry_passes +from trtmc_benchmark.artifact_metrics import ( + metric_geometry_metrics, + metric_geometry_passes, + robot_action_metrics, + robot_action_passes, +) from .catalog import QualificationCase, QualificationError, load_benchmark from .datasets import Dataset, resolve_dataset @@ -70,12 +75,107 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _stereo_disparity_parity(case, context, dataset, output) elif metric_name == "metric_geometry_parity": result = _metric_geometry_parity(case, context, dataset, output) + elif metric_name == "robot_action_parity": + result = _robot_action_parity(case, context, dataset, output) else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) return result +def _robot_control_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("robot-control dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"robot-control dataset requires at least {sample_limit} requests") + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"robot-control request {index} must be an object") + sample = {"sample_id": str(request.get("id") or f"sample-{index}")} + for source, target in (("image", "image_path"), ("state", "state_path")): + relative = request.get(source) + if not isinstance(relative, str) or not relative: + raise QualificationError(f"robot-control request {index} has no {source}") + path = (root / relative).resolve() + if not path.is_relative_to(root) or not path.is_file(): + raise QualificationError( + f"robot-control request {index} {source} is unavailable: {path}" + ) + sample[target] = str(path) + selected.append(sample) + return selected + + +def _robot_action_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _robot_control_samples(dataset, sample_limit) + expected = _family_image_reference(case, context, output, selected) + actual, bundle = _candidate_outputs( + case, + context, + output, + "control", + [ + { + "sample_id": sample["sample_id"], + "request": { + "image_path": sample["image_path"], + "state_path": sample["state_path"], + }, + } + for sample in selected + ], + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("robot-action gate must be an object") + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("robot-action min_sample_pass_rate must be in [0, 1]") + thresholds = {key: value for key, value in gate.items() if key != "min_sample_pass_rate"} + rows = [] + for sample, candidate_sample, reference_sample in zip( + selected, actual, expected, strict=True + ): + try: + metrics = robot_action_metrics(candidate_sample, reference_sample) + passed = robot_action_passes(metrics, thresholds) + except ValueError as error: + raise QualificationError( + f"robot-action sample {sample['sample_id']} is invalid: {error}" + ) from error + rows.append({"sample_id": sample["sample_id"], "passed": passed, **metrics}) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": pass_rate, + **{name: max(float(row[name]) for row in rows) for name in thresholds}, + }, + "gate": dict(gate), + "samples": rows, + } + + def _metric_geometry_parity( case: QualificationCase, context: RuntimeContext, diff --git a/tools/benchmark_qualification/benchmarks/robot_action_parity.yaml b/tools/benchmark_qualification/benchmarks/robot_action_parity.yaml new file mode 100644 index 0000000000..68ca3fed63 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/robot_action_parity.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: lerobot-act-recorded-observation + version: episode-0-frame-0-v1 + sha256: 33eb2456e87abbe65566c186b21fade3a410a3d230fe0fee96009077ca70181a + source: + mode: family + path: ../data/benchmark_robot_control.json + license: Apache-2.0 +selection: + method: first +input: + adapter: image_state_paths +metric: + name: robot_action_parity diff --git a/tools/benchmark_qualification/benchmarks/robot_action_performance.yaml b/tools/benchmark_qualification/benchmarks/robot_action_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/robot_action_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index 2468efff73..d4ca6a0938 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -43,6 +43,8 @@ from trtmc_benchmark.artifact_metrics import ( # noqa: E402 metric_geometry_metrics, metric_geometry_passes, + robot_action_metrics, + robot_action_passes, ) from trtmc_benchmark.catalog import ManifestCatalog, resolve_case, selected_task_for_case # noqa: E402 from trtmc_benchmark.task_adapters import default_operation # noqa: E402 @@ -94,6 +96,7 @@ "prompted-mask-parity", "reranking-order", "robot-action-shape", + "robot-action-parity", "segmentation-shape", "transcription-text", "vision-language-text", @@ -1469,6 +1472,23 @@ def _output_contract( and right.get("finite") is True ) return matched, "robot action output contract differs" if not matched else "", None + if contract == "robot-action-parity": + try: + metrics = robot_action_metrics(left, right) + passed = robot_action_passes(metrics, entry.spec["baseline"]) + except ValueError as error: + return ( + False, + f"robot action parity could not be evaluated: {error}", + {"contract": contract, "numerical_parity_checked": True}, + ) + evidence = { + "contract": contract, + "numerical_parity_checked": True, + **metrics, + } + reason = "robot action numerical parity is outside the contract" if not passed else "" + return passed, reason, evidence if contract == "disparity-parity": evidence = _disparity(entry, left, right) return bool(evidence["passed"]), str(evidence.get("reason", "")), evidence diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index ed264fc2ff..235d76e7cd 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -529,16 +529,22 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", - "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", - "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", + "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", + "tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml", + "tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml", + "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", "tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml", - "tools/benchmark_qualification/benchmarks/reranking_performance.yaml", + "tools/benchmark_qualification/benchmarks/reranking_performance.yaml", + "tools/benchmark_qualification/benchmarks/robot_action_parity.yaml", + "tools/benchmark_qualification/benchmarks/robot_action_performance.yaml", "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", - "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", + "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", + "tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml", + "tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml", "tools/benchmark_qualification/benchmarks/vision_language_performance.yaml", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 5402f2e235..1c352c11a0 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -420,6 +420,108 @@ def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> N assert resolved == {"image_path": str(image.resolve()), "batch_size": 1} +def test_robot_action_accuracy_compares_complete_action_chunk( + tmp_path: Path, monkeypatch +) -> None: + data = tmp_path / "families/lerobot_act/tests/data" + data.mkdir(parents=True) + image = data / "image.png" + state = data / "state.f32" + image.write_bytes(b"fixture") + state.write_bytes(b"state") + dataset_path = data / "requests.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + {"id": "recorded", "image": image.name, "state": state.name} + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/lerobot_act/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="act-example", + family="lerobot_act", + name="recorded-action-parity", + benchmark="robot_action_parity", + candidate={ + "family": "lerobot_act", + "checkpoint": "lerobot/example", + "task": "robot_control", + "precision": "fp32", + "build": {}, + }, + values={ + "samples": 1, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "action_max_abs_error": 0.00005, + "action_mean_abs_error": 0.000005, + "action_rmse": 0.00001, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("lerobot-act-recorded-observation", dataset_path, "family", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + reference = { + "action_steps": 2, + "action_dim": 2, + "action_values": 4, + "actions": [0.1, -0.2, 0.3, -0.4], + "finite": True, + } + candidate = { + **reference, + "within_training_bounds": True, + } + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "robot_action_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, + "_family_image_reference", + lambda *_args: [reference], + ) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ([candidate], tmp_path / "act.bundle"), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["sample_pass_rate"] == 1.0 + assert result["metrics"]["action_max_abs_error"] == pytest.approx(0.0) + + def test_object_detection_accuracy_matches_classes_boxes_and_scores( tmp_path: Path, monkeypatch ) -> None: From e8f57be6427237e4adffd222450ad7807fd46378 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 00:38:05 +0000 Subject: [PATCH 057/100] refactor(test): simplify fake task factory Move fake task selection into a small helper so the runtime test factory stays within the repository cyclomatic-complexity limit without changing its mismatch coverage. Signed-off-by: chaofengw --- core/runtime/tests/fake_family.cpp | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 7c82c19fe0..7373e4d72f 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -65,19 +65,7 @@ class FakeSegmentation final : public trtmc::ISegmentation { } }; -} // namespace - -extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { - if (context.reader.info().family != "fake") - throw std::runtime_error("unexpected family"); - const char* backend_name = context.backend.name(); - if (backend_name == nullptr || - (std::string(backend_name) != "fake" && std::string(backend_name) != "trt_rtx")) { - throw std::runtime_error("unexpected backend"); - } - const auto plan = context.reader.read_section("engine.plan"); - if (std::string(plan.begin(), plan.end()) != "PLAN") - throw std::runtime_error("unexpected engine plan"); +trtmc::ITask* create_fake_task(const trtmc::FamilyContext& context) { if (context.reader.info().task == trtmc::IEncoding::kTask) return new FakeEncoding(); if (context.reader.info().task == trtmc::IEmbedding::kTask) @@ -92,3 +80,19 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context } throw std::runtime_error("unsupported fake task"); } + +} // namespace + +extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { + if (context.reader.info().family != "fake") + throw std::runtime_error("unexpected family"); + const char* backend_name = context.backend.name(); + if (backend_name == nullptr || + (std::string(backend_name) != "fake" && std::string(backend_name) != "trt_rtx")) { + throw std::runtime_error("unexpected backend"); + } + const auto plan = context.reader.read_section("engine.plan"); + if (std::string(plan.begin(), plan.end()) != "PLAN") + throw std::runtime_error("unexpected engine plan"); + return create_fake_task(context); +} From 042d6e591b7df7fb44e0aa67d3c087aed0e89137 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 00:40:48 +0000 Subject: [PATCH 058/100] style(test): format fake segmentation signature Signed-off-by: chaofengw --- core/runtime/tests/fake_family.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 7373e4d72f..c1b90516f1 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -58,8 +58,7 @@ class FakeEmbedding final : public trtmc::IEmbedding { class FakeSegmentation final : public trtmc::ISegmentation { public: - trtmc::SegmentResult segment(const float*, std::int32_t height, - std::int32_t width) override { + trtmc::SegmentResult segment(const float*, std::int32_t height, std::int32_t width) override { return {std::vector(static_cast(height) * width, 7), height, width}; } From b5bcc7b6dd0cb8ab4d017e491f29d6fda46f8607 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 00:59:32 +0000 Subject: [PATCH 059/100] fix(benchmark): align execution contracts Match DeepSeek OCR performance timing to its family-owned pipeline reference. Load LocateAnything tokenizer framing from tokenizer.json so candidate builds do not depend on Transformers tokenizer constructor behavior. Signed-off-by: chaofengw --- .../tests/benchmark/deepseek-ocr.yaml | 2 +- families/locateanything/model.py | 12 +++----- .../locateanything/tests/test_hf_reference.py | 30 +++++++++++++++++++ .../vision_language_pipeline_performance.yaml | 15 ++++++++++ tools/tests/test_architecture.py | 1 + 5 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml diff --git a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml index 59c485676e..f6f6817b3f 100644 --- a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml +++ b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml @@ -35,7 +35,7 @@ accuracy: performance: - name: generate - benchmark: vision_language_performance + benchmark: vision_language_pipeline_performance operation: generate request: image_path: ../data/orc_test_img.jpeg diff --git a/families/locateanything/model.py b/families/locateanything/model.py index 144218cd7b..a0917a6cf0 100644 --- a/families/locateanything/model.py +++ b/families/locateanything/model.py @@ -338,15 +338,11 @@ def _detect_layer_prefix(readers) -> str: def _tokenizer_runtime_contract(model_dir: Path) -> dict[str, object]: """Resolve this family's exact native-tokenizer framing.""" - from transformers import AutoTokenizer + from tokenizers import Tokenizer - tokenizer = AutoTokenizer.from_pretrained( - str(model_dir), - trust_remote_code=True, - use_fast=True, - ) - default_ids = list(tokenizer.encode("hello")) - plain_ids = list(tokenizer.encode("hello", add_special_tokens=False)) + tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json")) + default_ids = list(tokenizer.encode("hello").ids) + plain_ids = list(tokenizer.encode("hello", add_special_tokens=False).ids) if default_ids == plain_ids: prefix_ids, suffix_ids = [], [] elif not plain_ids: diff --git a/families/locateanything/tests/test_hf_reference.py b/families/locateanything/tests/test_hf_reference.py index e8b24c8c04..0630d217a4 100644 --- a/families/locateanything/tests/test_hf_reference.py +++ b/families/locateanything/tests/test_hf_reference.py @@ -4,8 +4,11 @@ from __future__ import annotations import inspect +import sys from pathlib import Path +from types import SimpleNamespace +from families.locateanything import model from families.locateanything.tests import hf_reference, vision_oracle @@ -45,3 +48,30 @@ def test_hf_reference_is_local_manual_and_slow() -> None: assert 'generation_mode="slow"' in source assert "preprocess_image_inputs_for_trt(" in source assert "local_files_only=True" in source + + +def test_builder_uses_tokenizer_json_for_native_framing(monkeypatch, tmp_path) -> None: + calls: list[str] = [] + + class Encoding: + def __init__(self, ids: list[int]) -> None: + self.ids = ids + + class LocalTokenizer: + def encode(self, _text: str, add_special_tokens: bool = True) -> Encoding: + return Encoding([1, 7, 2] if add_special_tokens else [7]) + + class Tokenizer: + @staticmethod + def from_file(path: str) -> LocalTokenizer: + calls.append(path) + return LocalTokenizer() + + monkeypatch.setitem(sys.modules, "tokenizers", SimpleNamespace(Tokenizer=Tokenizer)) + + assert model._tokenizer_runtime_contract(tmp_path) == { + "tokenizer_add_special_tokens": False, + "tokenizer_prefix_ids": [1], + "tokenizer_suffix_ids": [2], + } + assert calls == [str(tmp_path / "tokenizer.json")] diff --git a/tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml b/tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml new file mode 100644 index 0000000000..d12095f992 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: true +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 235d76e7cd..3a50bd948b 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -547,6 +547,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml", "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml", + "tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml", "tools/benchmark_qualification/benchmarks/vision_language_performance.yaml", "tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", From ba49e8ccefdc0e5488615fdcfbd3bdb21ba40cac Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 04:28:12 +0000 Subject: [PATCH 060/100] fix(benchmark): retain perception outputs Serialize complete legacy detection and prompted-segmentation results so Accuracy and Performance can compare the public task outputs instead of treating successful candidate execution as an infrastructure error. Add native fake-family coverage for both contracts. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 31 ++++++++++-- .../native/test_benchmark_worker_e2e.cpp | 47 +++++++++++++++++++ core/runtime/tests/fake_family.cpp | 42 ++++++++++++++++- 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 27bda34121..ab56d6347b 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -503,9 +503,19 @@ Json run_segment_prompted(trtmc::ITask& task, const Json& request, const Timing& }; } return measure(timing, invoke, [](const trtmc::PromptedSegmentationResult& result) { - return Json{{"segmented_images", 1}, {"generated_masks", result.num_masks}, - {"num_masks", result.num_masks}, {"height", result.height}, - {"width", result.width}, {"mask_pixels", result.masks.size()}}; + if (result.boxes.size() % 4U != 0U) + throw std::runtime_error("prompted segmentation returned incomplete boxes"); + Json boxes = Json::array(); + for (std::size_t offset = 0; offset < result.boxes.size(); offset += 4U) { + boxes.push_back({result.boxes[offset], result.boxes[offset + 1U], + result.boxes[offset + 2U], result.boxes[offset + 3U]}); + } + return Json{ + {"segmented_images", 1}, {"generated_masks", result.num_masks}, + {"num_masks", result.num_masks}, {"height", result.height}, + {"width", result.width}, {"mask_pixels", result.masks.size()}, + {"masks", result.masks}, {"iou_scores", result.iou_scores}, + {"boxes", std::move(boxes)}, {"box_coordinates", "original_image_pixels_xyxy"}}; }); } @@ -527,10 +537,23 @@ Json run_detect(trtmc::ITask& task, const Json& request, const Timing& timing) { return measure( timing, [&]() { return interface.detect(image.pixels.data(), image.height, image.width); }, [](const trtmc::ObjectDetectionResult& result) { + Json boxes = Json::array(), scores = Json::array(), classes = Json::array(); + for (const auto& detection : result.boxes) { + boxes.insert(boxes.end(), + {detection.x_min, detection.y_min, detection.x_max, detection.y_max}); + scores.push_back(detection.score); + classes.push_back(detection.class_id); + } return Json{{"detected_images", 1}, {"detections", result.boxes.size()}, {"image_height", result.image_height}, - {"image_width", result.image_width}}; + {"image_width", result.image_width}, + {"boxes", std::move(boxes)}, + {"scores", std::move(scores)}, + {"class_ids", std::move(classes)}, + {"shape", {result.boxes.size(), 4}}, + {"coordinates", "xyxy"}, + {"units", "pixels"}}; }); } diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index 7d2275e340..01186dbb92 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -183,6 +183,53 @@ int main(int argc, char** argv) { check(segment_result.at("output_summary").at("mask") == Json::array({7, 7}), "legacy segmentation retains labels for Accuracy comparison"); std::filesystem::remove(segment_bundle); + + const auto detection_bundle = runtime_root / "benchmark_fake_detection.bundle"; + write_bundle(detection_bundle, "object_detection"); + Json detection_request = request; + detection_request["case_name"] = "fake-detection"; + detection_request["bundle"] = detection_bundle.string(); + detection_request["operation"] = "detect"; + detection_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << detection_request << '\n'; + } + check(std::system(command.c_str()) == 0, "detection worker process completed"); + std::ifstream detection_output_file(output_path); + Json detection_result; + detection_output_file >> detection_result; + const auto& detection = detection_result.at("output_summary"); + check(detection.at("boxes") == Json::array({-2, 1, 5, 1, 0, 0, 1, 1}) && + detection.at("scores") == Json::array({0.75, 0.0}) && + detection.at("class_ids") == Json::array({42, 7}) && + detection.at("shape") == Json::array({2, 4}) && + detection.at("coordinates") == "xyxy" && detection.at("units") == "pixels", + "legacy detection retains boxes, scores and classes for Accuracy comparison"); + std::filesystem::remove(detection_bundle); + + const auto prompted_bundle = runtime_root / "benchmark_fake_prompted.bundle"; + write_bundle(prompted_bundle, "prompted_segmentation"); + Json prompted_request = request; + prompted_request["case_name"] = "fake-prompted-segmentation"; + prompted_request["bundle"] = prompted_bundle.string(); + prompted_request["operation"] = "segment_prompted"; + prompted_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << prompted_request << '\n'; + } + check(std::system(command.c_str()) == 0, "prompted segmentation worker process completed"); + std::ifstream prompted_output_file(output_path); + Json prompted_result; + prompted_output_file >> prompted_result; + const auto& prompted = prompted_result.at("output_summary"); + check(prompted.at("masks") == Json::array({1, -1, -2, 2}) && + prompted.at("iou_scores") == Json::array({0.5, 0.75}) && + prompted.at("boxes") == Json::array({{0, 0, 1, 1}, {1, 0, 2, 1}}) && + prompted.at("box_coordinates") == "original_image_pixels_xyxy", + "legacy prompted segmentation retains masks, scores and boxes for comparison"); + std::filesystem::remove(prompted_bundle); std::filesystem::remove(legacy_image); Json sdk = request; diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index c1b90516f1..5172fe2f4b 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -64,13 +64,51 @@ class FakeSegmentation final : public trtmc::ISegmentation { } }; +class FakeObjectDetection final : public trtmc::IObjectDetection { + public: + trtmc::ObjectDetectionResult detect(const float*, std::int32_t height, + std::int32_t width) override { + trtmc::ObjectDetectionResult result; + result.boxes = {{-2.0F, 1.0F, 5.0F, 1.0F, 0.75F, 42}, {0.0F, 0.0F, 1.0F, 1.0F, 0.0F, 7}}; + result.image_height = height; + result.image_width = width; + return result; + } +}; + +class FakePointPromptedSegmentation final : public trtmc::IPointPromptedSegmentation { + public: + trtmc::PromptedSegmentationResult segment_prompted(const float*, std::int32_t height, + std::int32_t width, float, float, + bool) override { + trtmc::PromptedSegmentationResult result; + result.masks = {1.0F, -1.0F, -2.0F, 2.0F}; + result.iou_scores = {0.5F, 0.75F}; + result.boxes = {0.0F, 0.0F, 1.0F, 1.0F, 1.0F, 0.0F, 2.0F, 1.0F}; + result.num_masks = 2; + result.height = height; + result.width = width; + return result; + } +}; + +trtmc::ITask* create_fake_perception_task(const trtmc::FamilyContext& context) { + if (context.reader.info().task == trtmc::ISegmentation::kTask) + return new FakeSegmentation(); + if (context.reader.info().task == trtmc::IObjectDetection::kTask) + return new FakeObjectDetection(); + if (context.reader.info().task == trtmc::IPointPromptedSegmentation::kTask) + return new FakePointPromptedSegmentation(); + return nullptr; +} + trtmc::ITask* create_fake_task(const trtmc::FamilyContext& context) { + if (auto* task = create_fake_perception_task(context)) + return task; if (context.reader.info().task == trtmc::IEncoding::kTask) return new FakeEncoding(); if (context.reader.info().task == trtmc::IEmbedding::kTask) return new FakeEmbedding(); - if (context.reader.info().task == trtmc::ISegmentation::kTask) - return new FakeSegmentation(); if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) return new FakeForecast(context.backend, context.kv_cache_size_bytes); if (context.reader.info().task == trtmc::ITextGeneration::kTask) { From fc91f8bcb0572d63c28394a76c0f6393d89867c2 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 04:38:02 +0000 Subject: [PATCH 061/100] fix(benchmark): keep YOLO NMS in inference mode Run Ultralytics non-maximum suppression in the same inference scope as the reference forward pass. This prevents inference tensors from being mutated outside inference mode in both YOLO11 and YOLOv8 Accuracy and Performance references. Signed-off-by: chaofengw --- families/yolo11/tests/benchmark/reference.py | 14 +++++++------- families/yolov8/tests/benchmark/reference.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/families/yolo11/tests/benchmark/reference.py b/families/yolo11/tests/benchmark/reference.py index 63e5b440df..2647d54d08 100644 --- a/families/yolo11/tests/benchmark/reference.py +++ b/families/yolo11/tests/benchmark/reference.py @@ -130,13 +130,13 @@ def _detect( with torch.inference_mode(): raw = model(pixels) - raw = raw[0] if isinstance(raw, (list, tuple)) else raw - kept = non_max_suppression( - raw, - conf_thres=threshold, - iou_thres=0.7, - max_det=300, - )[0] + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + kept = non_max_suppression( + raw, + conf_thres=threshold, + iou_thres=0.7, + max_det=300, + )[0] rows = kept.detach().float().cpu().tolist() scale = float(geometry["scale"]) pad_x = int(geometry["pad_x"]) diff --git a/families/yolov8/tests/benchmark/reference.py b/families/yolov8/tests/benchmark/reference.py index 1a37c75797..b84140ffa2 100644 --- a/families/yolov8/tests/benchmark/reference.py +++ b/families/yolov8/tests/benchmark/reference.py @@ -130,13 +130,13 @@ def _detect( with torch.inference_mode(): raw = model(pixels) - raw = raw[0] if isinstance(raw, (list, tuple)) else raw - kept = non_max_suppression( - raw, - conf_thres=threshold, - iou_thres=0.7, - max_det=300, - )[0] + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + kept = non_max_suppression( + raw, + conf_thres=threshold, + iou_thres=0.7, + max_det=300, + )[0] rows = kept.detach().float().cpu().tolist() scale = float(geometry["scale"]) pad_x = int(geometry["pad_x"]) From b68b9da9eda5e2aa421bd0565972c9e3375ae084 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 06:04:14 +0000 Subject: [PATCH 062/100] fix(canary): declare benchmark source language Signed-off-by: chaofengw --- families/canary/tests/benchmark/canary-1b-v2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/families/canary/tests/benchmark/canary-1b-v2.yaml b/families/canary/tests/benchmark/canary-1b-v2.yaml index a0d5521132..5cbef36fa0 100644 --- a/families/canary/tests/benchmark/canary-1b-v2.yaml +++ b/families/canary/tests/benchmark/canary-1b-v2.yaml @@ -20,6 +20,7 @@ accuracy: benchmark: librispeech_transcription samples: 10 request: &request + language: en max_new_tokens: 50 reference: command: reference.py From 72ed6545facb268831e492bd18514c42e76f4176 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 06:47:50 +0000 Subject: [PATCH 063/100] fix(benchmark): align qualification task inputs Signed-off-by: chaofengw --- .../moge/tests/benchmark/moge-2-vitl.yaml | 1 + .../benchmark/nemotron-labs-diffusion-8b.yaml | 2 +- .../tests/benchmark/reference.py | 33 +++++++- .../tests/test_benchmark_reference.py | 28 +++++++ tools/benchmark_qualification/catalog.py | 3 + tools/benchmark_qualification/runtime.py | 17 ++++ tools/tests/test_model_benchmark.py | 83 +++++++++++++++++++ 7 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 families/nemotron_labs_diffusion/tests/test_benchmark_reference.py diff --git a/families/moge/tests/benchmark/moge-2-vitl.yaml b/families/moge/tests/benchmark/moge-2-vitl.yaml index f38c24b510..fc44014ffd 100644 --- a/families/moge/tests/benchmark/moge-2-vitl.yaml +++ b/families/moge/tests/benchmark/moge-2-vitl.yaml @@ -8,6 +8,7 @@ candidate: checkpoint: Ruicheng/moge-2-vitl revision: 39c4d5e957afe587e04eec59dc2bcc3be5ecd968 task: monocular_geometry + selected_task: image_to_metric_geometry precision: fp32 bundle: moge-2-vitl.bundle build: diff --git a/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml b/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml index f77a8c23b7..886ffff711 100644 --- a/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml +++ b/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml @@ -37,7 +37,7 @@ accuracy: use_chat_template: true enable_thinking: false do_sample: false - generation_mode: ar + text_generation_mode: ar gate: min_pass_rate: 0.9 allowed_failures: 1 diff --git a/families/nemotron_labs_diffusion/tests/benchmark/reference.py b/families/nemotron_labs_diffusion/tests/benchmark/reference.py index a30a4d1d47..5d40a3ae4e 100644 --- a/families/nemotron_labs_diffusion/tests/benchmark/reference.py +++ b/families/nemotron_labs_diffusion/tests/benchmark/reference.py @@ -38,6 +38,22 @@ def _inputs(tokenizer: Any, prompt: str, generation: dict[str, Any]) -> Any: return tokenizer(rendered, return_tensors="pt", add_special_tokens=False) +def _truncate(tokenizer: Any, prompt: str, limit: int, side: str) -> str: + if limit < 1 or side not in {"left", "right"}: + raise ValueError("prompt token limit and truncation side are invalid") + token_ids = [int(value) for value in tokenizer.encode(prompt, add_special_tokens=False)] + if len(token_ids) <= limit: + return prompt + selected = token_ids[-limit:] if side == "left" else token_ids[:limit] + return str( + tokenizer.decode( + selected, + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + ) + ) + + def main(argv: Sequence[str] | None = None) -> int: arguments = _parser().parse_args(argv) request = json.loads(arguments.request.read_text(encoding="utf-8")) @@ -45,8 +61,9 @@ def main(argv: Sequence[str] | None = None) -> int: generation = request.get("generation") if not isinstance(samples, list) or not samples or not isinstance(generation, dict): raise ValueError("reference request must contain samples and generation") - if generation.get("generation_mode", "ar") != "ar": - raise ValueError("this qualification reference supports generation_mode=ar") + mode = generation.get("text_generation_mode", generation.get("generation_mode", "ar")) + if mode != "ar": + raise ValueError("this qualification reference supports text_generation_mode=ar") import torch from transformers import AutoModel, AutoTokenizer @@ -64,9 +81,17 @@ def main(argv: Sequence[str] | None = None) -> int: **options, ).eval().to("cuda") max_new_tokens = int(generation.get("max_new_tokens", 20)) + prompt_token_limit = int(request.get("prompt_token_limit", 192)) + truncation_side = str(request.get("truncation_side", "left")) results = [] for sample in samples: - encoded = _inputs(tokenizer, str(sample["prompt"]), generation) + prompt = _truncate( + tokenizer, + str(sample["prompt"]), + prompt_token_limit, + truncation_side, + ) + encoded = _inputs(tokenizer, prompt, generation) input_ids = encoded["input_ids"].to(model.device) with torch.inference_mode(): generated = model.ar_generate( @@ -81,7 +106,7 @@ def main(argv: Sequence[str] | None = None) -> int: results.append( { "sample_id": str(sample["sample_id"]), - "prompt": str(sample["prompt"]), + "prompt": prompt, "token_ids": token_ids, "text": tokenizer.decode(token_ids, skip_special_tokens=True), } diff --git a/families/nemotron_labs_diffusion/tests/test_benchmark_reference.py b/families/nemotron_labs_diffusion/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..94a0861a34 --- /dev/null +++ b/families/nemotron_labs_diffusion/tests/test_benchmark_reference.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from families.nemotron_labs_diffusion.tests.benchmark.reference import _truncate + + +class _Tokenizer: + def encode(self, prompt: str, *, add_special_tokens: bool) -> list[int]: + assert add_special_tokens is False + return [int(value) for value in prompt.split()] + + def decode( + self, + token_ids: list[int], + *, + skip_special_tokens: bool, + clean_up_tokenization_spaces: bool, + ) -> str: + assert skip_special_tokens is False + assert clean_up_tokenization_spaces is False + return " ".join(str(value) for value in token_ids) + + +def test_benchmark_reference_applies_declared_prompt_window() -> None: + tokenizer = _Tokenizer() + + assert _truncate(tokenizer, "1 2 3 4", 2, "left") == "3 4" + assert _truncate(tokenizer, "1 2 3 4", 2, "right") == "1 2" diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 280923fe14..944c837170 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -132,6 +132,9 @@ def _candidate(raw: Any, family: str, path: Path) -> dict[str, Any]: ) for field in ("checkpoint", "task", "precision"): _string(value.get(field), f"candidate.{field}", path) + selected_task = value.get("selected_task") + if selected_task is not None: + _string(selected_task, "candidate.selected_task", path) revision = value.get("revision") if revision is not None and not isinstance(revision, str): raise QualificationError(f"{path}: candidate.revision must be a string") diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 7996590fb2..436b78068f 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -23,6 +23,16 @@ REPOSITORY = Path(__file__).resolve().parents[2] +# Public benchmark requests use Task API field names. These legacy manifest +# tasks still describe their primary assets under ``inputs`` with shorter names. +_MANIFEST_INPUT_FIELDS = { + "robot_control": {"image_path": "image", "state_path": "state"}, + "stereo_disparity": { + "left_image_path": "left_image", + "right_image_path": "right_image", + }, +} + @dataclass(frozen=True) class RuntimeContext: @@ -101,8 +111,15 @@ def write_model_descriptor( testcase: dict[str, Any] = {"name": case.name} if task == "time_series_forecast": testcase["inputs"] = dict(request) + elif task in _MANIFEST_INPUT_FIELDS: + testcase.update(request) + fields = _MANIFEST_INPUT_FIELDS[task] + testcase["inputs"] = {target: testcase.pop(source) for source, target in fields.items()} else: testcase.update(request) + selected_task = candidate.get("selected_task") + if selected_task is not None: + testcase["selected_task"] = str(selected_task) value = { "name": case.model, "hf_id": str(candidate["checkpoint"]), diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 1c352c11a0..0f31cbf9ec 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -1873,6 +1873,89 @@ def test_candidate_descriptor_is_generated_from_public_build_inputs(tmp_path: Pa assert "tests/manifests" not in descriptor.read_text(encoding="utf-8") +def test_candidate_descriptor_preserves_family_selected_task(tmp_path: Path) -> None: + case = QualificationCase( + kind="performance", + model="geometry-model", + family="geometry", + name="geometry", + benchmark="geometry_performance", + candidate={ + "family": "geometry", + "checkpoint": "example/geometry", + "task": "legacy_geometry", + "selected_task": "image_to_metric_geometry", + "precision": "fp32", + "build": {}, + }, + values={}, + source=tmp_path / "geometry.yaml", + reference_requirements=None, + ) + + descriptor = write_model_descriptor(case, tmp_path, {"image_path": "/data/image.png"}) + testcase = json.loads(descriptor.read_text(encoding="utf-8"))["testcases"][0] + + assert testcase["selected_task"] == "image_to_metric_geometry" + assert testcase["image_path"] == "/data/image.png" + + +@pytest.mark.parametrize( + ("task", "public_request", "expected_inputs", "expected_controls"), + [ + ( + "stereo_disparity", + { + "left_image_path": "/data/left.png", + "right_image_path": "/data/right.png", + "height": 700, + "width": 700, + }, + {"left_image": "/data/left.png", "right_image": "/data/right.png"}, + {"height": 700, "width": 700}, + ), + ( + "robot_control", + {"image_path": "/data/observation.png", "state_path": "/data/state.f32"}, + {"image": "/data/observation.png", "state": "/data/state.f32"}, + {}, + ), + ], +) +def test_candidate_descriptor_uses_manifest_input_contract( + tmp_path: Path, + task: str, + public_request: dict[str, object], + expected_inputs: dict[str, str], + expected_controls: dict[str, object], +) -> None: + case = QualificationCase( + kind="performance", + model="example-model", + family="example", + name="case", + benchmark="example_performance", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": task, + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) + + descriptor = write_model_descriptor(case, tmp_path, public_request) + testcase = json.loads(descriptor.read_text(encoding="utf-8"))["testcases"][0] + + assert testcase["inputs"] == expected_inputs + for name, value in expected_controls.items(): + assert testcase[name] == value + assert not (set(public_request) & set(testcase["inputs"])) + + def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> None: completed = run_command( [ From f1ba8e66a418c1a544bbffbb67f697a863bc5392 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 07:52:51 +0000 Subject: [PATCH 064/100] fix(timm): use family-owned benchmark tasks Signed-off-by: chaofengw --- .../benchmark/convnext-tiny-in12k-ft-in1k.yaml | 2 +- .../tests/benchmark/densenet121-ra-in1k.yaml | 2 +- .../tests/benchmark/efficientnet-b0-ra-in1k.yaml | 2 +- .../tests/benchmark/ghostnet-100-in1k.yaml | 2 +- .../tests/benchmark/inception-v3-tv-in1k.yaml | 2 +- .../benchmark/inception-resnet-v2-tf-in1k.yaml | 2 +- .../tests/benchmark/inception-v4-tf-in1k.yaml | 2 +- .../tests/benchmark/mobilenetv2-100-ra-in1k.yaml | 2 +- .../benchmark/mobilenetv3-large-100-ra-in1k.yaml | 2 +- .../tests/benchmark/mobilenetv4-conv-large.yaml | 2 +- .../tests/benchmark/mobilenetv4-conv-medium.yaml | 2 +- .../benchmark/mobilenetv4-conv-small-050.yaml | 2 +- .../tests/benchmark/mobilenetv4-conv-small.yaml | 2 +- .../tests/benchmark/regnety-040-ra3-in1k.yaml | 2 +- .../tests/benchmark/resnest50d-in1k.yaml | 2 +- .../tests/benchmark/resnet50-a1-in1k.yaml | 2 +- .../swin-tiny-patch4-window7-224-ms-in1k.yaml | 2 +- ...imm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml | 2 +- .../benchmark/xcit-nano-12-p16-224-fb-in1k.yaml | 2 +- .../benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml | 2 +- tools/tests/test_model_benchmark.py | 15 +++++++++++++++ 21 files changed, 35 insertions(+), 20 deletions(-) diff --git a/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml b/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml index 4de61a7d04..f41ce9a242 100644 --- a/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml +++ b/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_convnext checkpoint: timm/convnext_tiny.in12k_ft_in1k revision: aa096f03029c7f0ec052013f64c819b34f8ad790 - task: classification + task: image_to_class_scores precision: fp16 bundle: convnext-tiny-in12k-ft-in1k.bundle build: diff --git a/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml index fce485638e..837a24e586 100644 --- a/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml +++ b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml @@ -6,7 +6,7 @@ model: densenet121-ra-in1k candidate: family: timm_densenet checkpoint: timm/densenet121.ra_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: densenet121-ra-in1k.bundle build: diff --git a/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml index 3e552f7f45..54d580fb30 100644 --- a/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml +++ b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml @@ -6,7 +6,7 @@ model: efficientnet-b0-ra-in1k candidate: family: timm_efficientnet checkpoint: timm/efficientnet_b0.ra_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: efficientnet-b0-ra-in1k.bundle build: diff --git a/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml b/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml index 6b3f39f18a..309e23494b 100644 --- a/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml +++ b/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_ghostnet checkpoint: timm/ghostnet_100.in1k revision: e524c2ee5d5a9412f802ce38ff895aba07b30910 - task: classification + task: image_to_class_scores precision: fp16 bundle: ghostnet-100-in1k.bundle build: diff --git a/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml index f5e1ff44b7..08011445ea 100644 --- a/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml +++ b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml @@ -6,7 +6,7 @@ model: inception-v3-tv-in1k candidate: family: timm_inception checkpoint: timm/inception_v3.tv_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: inception-v3-tv-in1k.bundle build: diff --git a/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml b/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml index 0a3c2461b5..fd0584220e 100644 --- a/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml +++ b/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_inception_resnet checkpoint: timm/inception_resnet_v2.tf_in1k revision: 836b07d7d599da247de92f148c7a07fda18afced - task: classification + task: image_to_class_scores precision: fp16 bundle: inception-resnet-v2-tf-in1k.bundle build: diff --git a/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml b/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml index 3dac070b5c..2869f6a1d0 100644 --- a/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml +++ b/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_inception_v4 checkpoint: timm/inception_v4.tf_in1k revision: ad5d294cda312745a9afa433e45d9ccd956a1548 - task: classification + task: image_to_class_scores precision: fp16 bundle: inception-v4-tf-in1k.bundle build: diff --git a/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml index d22fec0ab6..79b729f77a 100644 --- a/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml +++ b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml @@ -6,7 +6,7 @@ model: mobilenetv2-100-ra-in1k candidate: family: timm_mobilenetv2 checkpoint: timm/mobilenetv2_100.ra_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: mobilenetv2-100-ra-in1k.bundle build: diff --git a/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml index 8a0a338665..ae044e5f85 100644 --- a/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml +++ b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml @@ -6,7 +6,7 @@ model: mobilenetv3-large-100-ra-in1k candidate: family: timm_mobilenetv3 checkpoint: timm/mobilenetv3_large_100.ra_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: mobilenetv3-large-100-ra-in1k.bundle build: diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml index 99cef8fa63..893a2a9863 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml @@ -6,7 +6,7 @@ model: mobilenetv4-conv-large candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_large.e500_r256_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-large.bundle build: diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml index 538206be01..45065bde30 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml @@ -6,7 +6,7 @@ model: mobilenetv4-conv-medium candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_medium.e500_r256_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-medium.bundle build: diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml index 80a02ff7cc..4b2a6e84bc 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml @@ -6,7 +6,7 @@ model: mobilenetv4-conv-small-050 candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_small_050.e3000_r224_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-small-050.bundle build: diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml index e9bcd35a24..10e9af4330 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml @@ -6,7 +6,7 @@ model: mobilenetv4-conv-small candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_small.e2400_r224_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-small.bundle build: diff --git a/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml b/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml index 21b5b68cc1..869dde68ac 100644 --- a/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml +++ b/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_regnet checkpoint: timm/regnety_040.ra3_in1k revision: a1875159c7b04f3b3189af804642eebba1e2c118 - task: classification + task: image_to_class_scores precision: fp16 bundle: regnety-040-ra3-in1k.bundle build: diff --git a/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml b/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml index 6b1a866328..0b23a76c68 100644 --- a/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml +++ b/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_resnest checkpoint: timm/resnest50d.in1k revision: 2239b0d822126dee3bc7a5357514de35b8f42092 - task: classification + task: image_to_class_scores precision: fp16 bundle: resnest50d-in1k.bundle build: diff --git a/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml index 011118946e..b4a6d72b32 100644 --- a/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml +++ b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml @@ -6,7 +6,7 @@ model: resnet50-a1-in1k candidate: family: timm_resnet checkpoint: timm/resnet50.a1_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: resnet50-a1-in1k.bundle build: diff --git a/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml b/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml index def3334b85..d2cd79761b 100644 --- a/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml +++ b/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_swin checkpoint: timm/swin_tiny_patch4_window7_224.ms_in1k revision: 4cc3a7275b50b53a7bec45f32c236ebe64227cff - task: classification + task: image_to_class_scores precision: fp16 bundle: swin-tiny-patch4-window7-224-ms-in1k.bundle build: diff --git a/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml index 57b53110a8..36bf5a9db5 100644 --- a/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml +++ b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml @@ -6,7 +6,7 @@ model: timm-vit-base-p16-224-augreg-in21k-ft-in1k candidate: family: timm_vit checkpoint: timm/vit_base_patch16_224.augreg_in21k_ft_in1k - task: classification + task: image_to_class_scores precision: fp16 bundle: timm-vit-base-p16-224-augreg-in21k-ft-in1k.bundle build: diff --git a/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml b/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml index 50150de079..beaa2ebe46 100644 --- a/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml +++ b/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_xcit checkpoint: timm/xcit_nano_12_p16_224.fb_in1k revision: b181b4623913d78fd0927601c36e40b9d8bc74ce - task: classification + task: image_to_class_scores precision: fp16 bundle: xcit-nano-12-p16-224-fb-in1k.bundle build: diff --git a/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml b/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml index 4d8d2be5c8..2c5ed7c2f4 100644 --- a/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml +++ b/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml @@ -7,7 +7,7 @@ candidate: family: timm_xcit checkpoint: timm/xcit_tiny_12_p16_224.fb_in1k revision: 060449891838eaf301bdb3673e3daaf58e0370de - task: classification + task: image_to_class_scores precision: fp16 bundle: xcit-tiny-12-p16-224-fb-in1k.bundle build: diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 0f31cbf9ec..7c5fa26156 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -53,6 +53,21 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert nemotron_h and all(not case.reference_build_isolation for case in nemotron_h) +def test_timm_qualification_profiles_use_the_family_build_task() -> None: + cases = discover(REPOSITORY) + profiles = { + case.model: case + for case in cases + if case.kind == "accuracy" and case.family.startswith("timm_") + } + + assert profiles + for model, case in profiles.items(): + manifest = case.source.parent.parent / "manifests" / f"{model}.json" + declared = json.loads(manifest.read_text(encoding="utf-8")) + assert case.candidate["task"] == declared["task"], model + + def test_l0_configs_outside_the_benchmark_folder_are_not_discovered( tmp_path: Path, ) -> None: From 1a9b2122f1f9bb0750d56b8a252944883a9fa63f Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 09:05:06 +0000 Subject: [PATCH 065/100] fix(fast-foundation-stereo): recognize official checkpoint Signed-off-by: chaofengw --- families/fast_foundation_stereo/support.py | 1 + .../tests/test_support.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 families/fast_foundation_stereo/tests/test_support.py diff --git a/families/fast_foundation_stereo/support.py b/families/fast_foundation_stereo/support.py index 83cb7ffdbe..436caec435 100644 --- a/families/fast_foundation_stereo/support.py +++ b/families/fast_foundation_stereo/support.py @@ -8,6 +8,7 @@ describe = family_support( model_types=("fast-foundation-stereo", "fast_foundation_stereo", "foundation_stereo_lite"), + required_files=("model_best_bp2_serialize.pth",), tasks=("stereo_disparity",), default_task="stereo_disparity", ) diff --git a/families/fast_foundation_stereo/tests/test_support.py b/families/fast_foundation_stereo/tests/test_support.py new file mode 100644 index 0000000000..1d468e4209 --- /dev/null +++ b/families/fast_foundation_stereo/tests/test_support.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from tensorrt_model_connect.model_support import ModelMetadata + +from families.fast_foundation_stereo.support import describe + + +def test_official_checkpoint_without_hugging_face_config_is_supported() -> None: + support = describe(ModelMetadata({}, {}, ("model_best_bp2_serialize.pth",))) + + assert support is not None + assert support.default_task == "stereo_disparity" + + +def test_unrelated_rootless_checkpoint_is_not_supported() -> None: + assert describe(ModelMetadata({}, {}, ("model.pt",))) is None From b047eb3f56ca76565a084cda36957b63884d8513 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 09:16:39 +0000 Subject: [PATCH 066/100] fix(benchmark): support legacy monocular geometry Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 81 ++++++++++++++++--- .../native/test_benchmark_worker_e2e.cpp | 35 ++++++++ .../trtmc_benchmark/task_adapters.py | 3 +- .../trtmc_benchmark/tests/test_benchmark.py | 7 ++ core/runtime/tests/fake_family.cpp | 17 ++++ .../moge/tests/benchmark/moge-2-vitl.yaml | 1 - 6 files changed, 131 insertions(+), 13 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index ab56d6347b..17b72052c5 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -465,6 +465,75 @@ Json run_transcribe(trtmc::ITask& task, const Json& request, const Timing& timin }); } +template +void write_binary_artifact(const std::string& path, const T* values, std::uint64_t count) { + if (count > static_cast(std::numeric_limits::max()) / sizeof(T)) + throw std::runtime_error("binary artifact is too large"); + std::ofstream output(path, std::ios::binary); + output.exceptions(std::ios::badbit | std::ios::failbit); + output.write(reinterpret_cast(values), + static_cast(count * sizeof(T))); + output.close(); +} + +Json run_geometry(trtmc::ITask& task, const Json& request, const Timing& timing) { + auto& interface = require_interface(task, "IMonocularGeometry"); + const auto path = request.at("image_path").get(); + std::optional cached; + if (!timing.asset_loading_included) + cached = read_image(path); + auto invoke = [&]() { + std::optional loaded; + if (!cached) + loaded = read_image(path); + const auto& image = cached ? *cached : *loaded; + return interface.estimate_geometry(image.pixels.data(), image.height, image.width); + }; + std::optional last; + for (int index = 0; index < timing.warmup; ++index) + last = invoke(); + Json observations = Json::array(); + for (int index = 0; index < timing.iterations; ++index) { + last.reset(); + const auto started = Clock::now(); + auto result = invoke(); + const auto wall_ms = elapsed_ms(started); + last.emplace(std::move(result)); + observations.push_back({{"runtime_e2e_wall_ms", wall_ms}, + {"geometry_images", 1}, + {"geometry_pixels", last->depth.size()}}); + } + if (last->height <= 0 || last->width <= 0) + throw std::runtime_error("monocular geometry returned invalid dimensions"); + const auto pixels = static_cast(last->height) * last->width; + if (last->points.size() != pixels * 3 || last->depth.size() != pixels || + last->mask.size() != pixels) + throw std::runtime_error("monocular geometry returned inconsistent output shapes"); + const auto prefix = request.at("_artifact_prefix").get(); + write_binary_artifact(prefix + ".points.f32", last->points.data(), last->points.size()); + write_binary_artifact(prefix + ".depth.f32", last->depth.data(), last->depth.size()); + write_binary_artifact(prefix + ".mask.u8", last->mask.data(), last->mask.size()); + Json intrinsics = Json::array(); + for (int row = 0; row < 3; ++row) + intrinsics.push_back({last->intrinsics[row * 3], last->intrinsics[row * 3 + 1], + last->intrinsics[row * 3 + 2]}); + return {{"observations", std::move(observations)}, + {"output_summary", + {{"geometry_images", 1}, + {"geometry_pixels", pixels}, + {"height", last->height}, + {"width", last->width}, + {"point_shape", {last->height, last->width, 3}}, + {"valid_pixels", std::count(last->mask.begin(), last->mask.end(), std::uint8_t{1})}, + {"normalized_intrinsics", std::move(intrinsics)}, + {"units", "meters"}, + {"camera_axes", {"right", "down", "forward"}}, + {"intrinsics_coordinates", "normalized_uv"}, + {"points_artifact", prefix + ".points.f32"}, + {"depth_artifact", prefix + ".depth.f32"}, + {"valid_mask_artifact", prefix + ".mask.u8"}}}}; +} + Json run_segment(trtmc::ITask& task, const Json& request, const Timing& timing) { auto& interface = require_interface(task, "ISegmentation"); const Image image = read_image(request.at("image_path").get()); @@ -2894,17 +2963,6 @@ Json run_track_pose(const trtmc::Model& model, const Json& request, const Timing }); } -template -void write_binary_artifact(const std::string& path, const T* values, std::uint64_t count) { - if (count > static_cast(std::numeric_limits::max()) / sizeof(T)) - throw std::runtime_error("binary artifact is too large"); - std::ofstream output(path, std::ios::binary); - output.exceptions(std::ios::badbit | std::ios::failbit); - output.write(reinterpret_cast(values), - static_cast(count * sizeof(T))); - output.close(); -} - Json run_geometry(const trtmc::Model& model, const Json& request, const Timing& timing, const std::string& task_id) { const auto task = task_for_operation(model, task_id); @@ -3796,6 +3854,7 @@ Json execute(const Json& request, const std::string& output_path) { {"classify", run_classify}, {"detect", run_detect}, {"extract_features", run_extract_features}, + {"geometry", run_geometry}, {"disparity", run_disparity}, {"rerank", run_rerank}, {"encode", run_encode}, diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index 01186dbb92..a291a6ba88 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -230,6 +230,41 @@ int main(int argc, char** argv) { prompted.at("box_coordinates") == "original_image_pixels_xyxy", "legacy prompted segmentation retains masks, scores and boxes for comparison"); std::filesystem::remove(prompted_bundle); + + const auto geometry_bundle = runtime_root / "benchmark_fake_geometry.bundle"; + write_bundle(geometry_bundle, "monocular_geometry"); + Json geometry_request = request; + geometry_request["case_name"] = "fake-monocular-geometry"; + geometry_request["bundle"] = geometry_bundle.string(); + geometry_request["operation"] = "geometry"; + geometry_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << geometry_request << '\n'; + } + check(std::system(command.c_str()) == 0, "monocular geometry worker process completed"); + std::ifstream geometry_output_file(output_path); + Json geometry_result; + geometry_output_file >> geometry_result; + const auto& geometry = geometry_result.at("output_summary"); + check(geometry_result.at("task") == "monocular_geometry" && + geometry_result.at("selected_task") == "monocular_geometry" && + geometry.at("geometry_pixels") == 2 && + geometry.at("point_shape") == Json::array({1, 2, 3}) && + geometry.at("valid_pixels") == 1 && + geometry.at("normalized_intrinsics") == + Json::array({{1.0, 0.0, 0.5}, {0.0, 1.0, 0.5}, {0.0, 0.0, 1.0}}) && + std::filesystem::file_size(geometry.at("points_artifact").get()) == + 6 * sizeof(float) && + std::filesystem::file_size(geometry.at("depth_artifact").get()) == + 2 * sizeof(float) && + std::filesystem::file_size( + geometry.at("valid_mask_artifact").get()) == 2, + "legacy monocular geometry retains complete metric artifacts"); + std::filesystem::remove(geometry.at("points_artifact").get()); + std::filesystem::remove(geometry.at("depth_artifact").get()); + std::filesystem::remove(geometry.at("valid_mask_artifact").get()); + std::filesystem::remove(geometry_bundle); std::filesystem::remove(legacy_image); Json sdk = request; diff --git a/apps/benchmark/trtmc_benchmark/task_adapters.py b/apps/benchmark/trtmc_benchmark/task_adapters.py index 598ec93013..3283d64548 100644 --- a/apps/benchmark/trtmc_benchmark/task_adapters.py +++ b/apps/benchmark/trtmc_benchmark/task_adapters.py @@ -58,6 +58,7 @@ class CaseResolution: "image_text_to_instance_masks": ("segment_prompted", 10, 100), "stereo_images_to_disparity": ("disparity", 3, 100), "image_to_metric_geometry": ("geometry", 3, 100), + "monocular_geometry": ("geometry", 3, 100), "text_to_pooled_features": ("encode", 50, 500), "text_to_head_scores": ("head_scores", 50, 500), "text_to_token_features": ("encode", 50, 500), @@ -312,7 +313,7 @@ def _request(task: str, case: Mapping[str, Any], root: Path) -> dict[str, Any]: return _text_controls(case, include_defaults=False) if task in _LATENT_TEXT | _LATENT_STEPS: return _latent_request(task, case, root) - if task == "image_to_metric_geometry": + if task in {"image_to_metric_geometry", "monocular_geometry"}: request = {"image_path": _semantic_asset(case, root, "image_path", "image", "test_image")} _copy_explicit(request, case, "fov_x") return request diff --git a/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py b/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py index 5960716926..7639a5ce42 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_benchmark.py @@ -340,6 +340,13 @@ def test_metric_geometry_has_an_image_input_and_preserves_optional_config(sdk_as } +def test_legacy_monocular_geometry_uses_the_same_public_request(sdk_assets: Path) -> None: + result = resolve_task_case("monocular_geometry", {"image": "image.ppm"}, sdk_assets) + + assert result.operation == "geometry" + assert result.request == {"image_path": str(sdk_assets / "image.ppm")} + + def test_image_only_detection_has_no_text_or_postprocessing_defaults(sdk_assets: Path) -> None: result = resolve_task_case("image_to_boxes", {"image": "image.ppm", "config": {"score": 0.0}}, sdk_assets) assert result.operation == "detect" diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 5172fe2f4b..f923cd40e5 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -76,6 +76,21 @@ class FakeObjectDetection final : public trtmc::IObjectDetection { } }; +class FakeMonocularGeometry final : public trtmc::IMonocularGeometry { + public: + trtmc::GeometryResult estimate_geometry(const float*, std::int32_t height, + std::int32_t width) override { + trtmc::GeometryResult result; + result.points = {1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F}; + result.depth = {3.0F, 6.0F}; + result.mask = {1, 0}; + result.intrinsics = {1.0F, 0.0F, 0.5F, 0.0F, 1.0F, 0.5F, 0.0F, 0.0F, 1.0F}; + result.height = height; + result.width = width; + return result; + } +}; + class FakePointPromptedSegmentation final : public trtmc::IPointPromptedSegmentation { public: trtmc::PromptedSegmentationResult segment_prompted(const float*, std::int32_t height, @@ -97,6 +112,8 @@ trtmc::ITask* create_fake_perception_task(const trtmc::FamilyContext& context) { return new FakeSegmentation(); if (context.reader.info().task == trtmc::IObjectDetection::kTask) return new FakeObjectDetection(); + if (context.reader.info().task == trtmc::IMonocularGeometry::kTask) + return new FakeMonocularGeometry(); if (context.reader.info().task == trtmc::IPointPromptedSegmentation::kTask) return new FakePointPromptedSegmentation(); return nullptr; diff --git a/families/moge/tests/benchmark/moge-2-vitl.yaml b/families/moge/tests/benchmark/moge-2-vitl.yaml index fc44014ffd..f38c24b510 100644 --- a/families/moge/tests/benchmark/moge-2-vitl.yaml +++ b/families/moge/tests/benchmark/moge-2-vitl.yaml @@ -8,7 +8,6 @@ candidate: checkpoint: Ruicheng/moge-2-vitl revision: 39c4d5e957afe587e04eec59dc2bcc3be5ecd968 task: monocular_geometry - selected_task: image_to_metric_geometry precision: fp32 bundle: moge-2-vitl.bundle build: From 33f284c268bed5c913658f173a0e37bfc8bbf35e Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 10:22:57 +0000 Subject: [PATCH 067/100] fix(benchmark): use prepared candidate model directories Signed-off-by: chaofengw --- families/fast_foundation_stereo/support.py | 2 +- .../benchmark/fast-foundation-stereo.yaml | 1 + .../tests/test_support.py | 6 +- tools/benchmark_qualification/accuracy.py | 2 +- tools/benchmark_qualification/catalog.py | 7 ++ tools/benchmark_qualification/performance.py | 2 +- tools/benchmark_qualification/runtime.py | 20 ++++- tools/tests/test_model_benchmark.py | 79 +++++++++++++++++++ 8 files changed, 112 insertions(+), 7 deletions(-) diff --git a/families/fast_foundation_stereo/support.py b/families/fast_foundation_stereo/support.py index 436caec435..6d255987d5 100644 --- a/families/fast_foundation_stereo/support.py +++ b/families/fast_foundation_stereo/support.py @@ -8,7 +8,7 @@ describe = family_support( model_types=("fast-foundation-stereo", "fast_foundation_stereo", "foundation_stereo_lite"), - required_files=("model_best_bp2_serialize.pth",), + required_files=("weights/23-36-37/model_best_bp2_serialize.pth",), tasks=("stereo_disparity",), default_task="stereo_disparity", ) diff --git a/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml b/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml index daa88cfffb..bb45d77647 100644 --- a/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml +++ b/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml @@ -7,6 +7,7 @@ candidate: family: fast_foundation_stereo checkpoint: nvidia/c-fast-foundationstereo revision: 9b446878c81ddb27593036767b29b2859d46103e + model_directory: trtmc-reference/Fast-FoundationStereo task: stereo_disparity precision: fp16 bundle: fast-foundation-stereo.bundle diff --git a/families/fast_foundation_stereo/tests/test_support.py b/families/fast_foundation_stereo/tests/test_support.py index 1d468e4209..04f0cf9ac5 100644 --- a/families/fast_foundation_stereo/tests/test_support.py +++ b/families/fast_foundation_stereo/tests/test_support.py @@ -6,8 +6,10 @@ from families.fast_foundation_stereo.support import describe -def test_official_checkpoint_without_hugging_face_config_is_supported() -> None: - support = describe(ModelMetadata({}, {}, ("model_best_bp2_serialize.pth",))) +def test_prepared_official_model_without_hugging_face_config_is_supported() -> None: + support = describe( + ModelMetadata({}, {}, ("weights/23-36-37/model_best_bp2_serialize.pth",)) + ) assert support is not None assert support.default_task == "stereo_disparity" diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 4aff731a2e..7a6235fe87 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -1236,7 +1236,7 @@ def _candidate_outputs( first_request = requests[0].get("request") if not isinstance(first_request, Mapping): raise QualificationError("Accuracy candidate request must be an object") - descriptor = write_model_descriptor(case, output, first_request) + descriptor = write_model_descriptor(case, output, first_request, context=context) bundle = prepare_bundle(case, context, output, descriptor) data_path = output / "candidate-inputs.json" _json(data_path, list(requests)) diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 944c837170..25e0355a78 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -135,6 +135,13 @@ def _candidate(raw: Any, family: str, path: Path) -> dict[str, Any]: selected_task = value.get("selected_task") if selected_task is not None: _string(selected_task, "candidate.selected_task", path) + model_directory = value.get("model_directory") + if model_directory is not None: + configured = Path(_string(model_directory, "candidate.model_directory", path)) + if configured.is_absolute() or ".." in configured.parts: + raise QualificationError( + f"{path}: candidate.model_directory must stay inside the family environment" + ) revision = value.get("revision") if revision is not None and not isinstance(revision, str): raise QualificationError(f"{path}: candidate.revision must be a string") diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py index 566e0d00be..3f1faa1d4c 100644 --- a/tools/benchmark_qualification/performance.py +++ b/tools/benchmark_qualification/performance.py @@ -78,7 +78,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st assert isinstance(measurement, Mapping) assert isinstance(reference_timing, Mapping) request = _resolve_family_assets(case, request) - descriptor = write_model_descriptor(case, output, request) + descriptor = write_model_descriptor(case, output, request, context=context) entry_id = f"qualification.{case.family}.{case.name}" suite = { "schema_version": "trtmc.perf-suite/v2", diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 436b78068f..92f1f5c68b 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -104,7 +104,11 @@ def require_candidate(context: RuntimeContext) -> tuple[Path, Path]: def write_model_descriptor( - case: QualificationCase, output: Path, request: Mapping[str, Any] + case: QualificationCase, + output: Path, + request: Mapping[str, Any], + *, + context: RuntimeContext | None = None, ) -> Path: candidate = case.candidate task = str(candidate["task"]) @@ -120,9 +124,21 @@ def write_model_descriptor( selected_task = candidate.get("selected_task") if selected_task is not None: testcase["selected_task"] = str(selected_task) + checkpoint = str(candidate["checkpoint"]) + configured_directory = candidate.get("model_directory") + if configured_directory is not None: + if context is None: + raise QualificationError("candidate.model_directory requires a runtime context") + environment = reference_python(case, context).parent.parent.resolve() + model_directory = (environment / str(configured_directory)).resolve() + if not model_directory.is_relative_to(environment) or not model_directory.is_dir(): + raise QualificationError( + f"prepared candidate model directory is unavailable: {model_directory}" + ) + checkpoint = str(model_directory) value = { "name": case.model, - "hf_id": str(candidate["checkpoint"]), + "hf_id": checkpoint, "hf_revision": str(candidate.get("revision", "")), "bundle": str(candidate.get("bundle", f"{case.model}.bundle")), "family": case.family, diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 7c5fa26156..0a47a08227 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -115,6 +115,30 @@ def test_trusted_remote_code_requires_an_immutable_revision( discover(tmp_path) +@pytest.mark.parametrize("model_directory", ["/tmp/model", "../model"]) +def test_candidate_model_directory_must_stay_in_the_family_environment( + tmp_path: Path, model_directory: str +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + f" model_directory: {model_directory}\n" + " task: classification\n" + " precision: fp16\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="must stay inside the family environment"): + discover(tmp_path) + + def test_one_model_file_owns_multiple_cases_without_testcase_indirection( tmp_path: Path, ) -> None: @@ -1915,6 +1939,61 @@ def test_candidate_descriptor_preserves_family_selected_task(tmp_path: Path) -> assert testcase["image_path"] == "/data/image.png" +def test_candidate_descriptor_uses_family_environment_model_directory( + tmp_path: Path, monkeypatch +) -> None: + environment = tmp_path / "family-environment" + python = environment / "bin/python" + python.parent.mkdir(parents=True) + python.write_text("", encoding="utf-8") + model_directory = environment / "prepared/model" + model_directory.mkdir(parents=True) + case = QualificationCase( + kind="accuracy", + model="source-model", + family="source_family", + name="parity", + benchmark="source_parity", + candidate={ + "family": "source_family", + "checkpoint": "example/source-model", + "model_directory": "prepared/model", + "task": "classification", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "source-model.yaml", + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + monkeypatch.setattr( + "tools.benchmark_qualification.runtime.reference_python", lambda *_args: python + ) + + descriptor = write_model_descriptor( + case, tmp_path, {"image_path": "/data/image.png"}, context=context + ) + + assert json.loads(descriptor.read_text(encoding="utf-8"))["hf_id"] == str( + model_directory.resolve() + ) + + @pytest.mark.parametrize( ("task", "public_request", "expected_inputs", "expected_controls"), [ From ea1d7fd75d92b24946539bdd02ae446387f33015 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 10:37:50 +0000 Subject: [PATCH 068/100] fix(benchmark): separate MoGe release scope Signed-off-by: chaofengw --- apps/benchmark/performance/release.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/benchmark/performance/release.yaml b/apps/benchmark/performance/release.yaml index 28ea3b5f9a..5e15dea3a7 100644 --- a/apps/benchmark/performance/release.yaml +++ b/apps/benchmark/performance/release.yaml @@ -18,6 +18,10 @@ defaults: asset_loading_included: false excluded_profiles: + - model: moge-2-vitl + reason: >- + Family-owned accuracy and internal performance qualification are present, + but no user-facing release-performance workload or receipt has been added. - model: s1-mini-fp16 reason: >- families/s1_mini has no release-performance workload or receipt yet; From ffb2b341221b684bfb575a903d2455627e800ab6 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 14:24:32 +0000 Subject: [PATCH 069/100] feat(benchmark): qualify remaining ready models Add family-owned Accuracy and Performance profiles for the remaining ready non-L0 models. Compare generated text, media, and audio through reusable task-output contracts and isolated reference environments. Signed-off-by: chaofengw --- .../baselines/sana_wm_reference.py | 22 +- .../performance/baselines/task_reference.py | 268 ++++++++++++++--- .../trtmc_benchmark/artifact_metrics.py | 186 +++++++++++- .../tests/test_artifact_metrics.py | 72 +++++ .../trtmc_benchmark/tests/test_perf_matrix.py | 6 +- families/bark/tests/benchmark/bark-large.yaml | 15 + families/bark/tests/benchmark/bark-small.yaml | 15 + .../flux/tests/benchmark/flux-2-dev-fp8.yaml | 30 ++ families/flux/tests/benchmark/flux-2-dev.yaml | 30 ++ .../flux/tests/benchmark/flux-schnell.yaml | 31 ++ .../gemma/tests/benchmark/gemma-3-1b.yaml | 33 +++ .../gemma/tests/benchmark/gemma-3-270m.yaml | 49 ++++ .../gemma/tests/benchmark/gemma-3-4b.yaml | 33 +++ .../tests/benchmark/lance-3b-x2t-image.yaml | 17 ++ .../tests/benchmark/prepare_environment.py | 45 +++ .../tests/benchmark/magpie-tts-357m.yaml | 15 + .../tests/benchmark/minimax-h3-768p.yaml | 22 ++ .../tests/benchmark/personaplex-7b.yaml | 17 ++ .../tests/benchmark/prepare_environment.py | 45 +++ .../tests/benchmark/pixart-sigma-1024.yaml | 35 +++ .../qwen/tests/benchmark/qwen3-0.6b-topp.yaml | 32 ++ .../qwen3-omni-30b-a3b-instruct.yaml | 15 + .../tests/benchmark/qwen-image-2512.yaml | 10 + .../tests/benchmark/qwen-image-edit-2511.yaml | 15 + .../tests/benchmark/qwen-image.yaml | 15 + .../tests/benchmark/prepare_environment.py | 50 ++++ .../benchmark/sana-wm-bidirectional.yaml | 33 +++ .../tests/benchmark/wan22-ti2v-5b.yaml | 15 + .../tests/benchmark/wan21-t2v-1.3b.yaml | 15 + .../tests/benchmark/z-image-turbo.yaml | 15 + tools/benchmark_qualification/accuracy.py | 276 +++++++++++++++--- .../audio_generation_model_performance.yaml | 14 + ...audio_generation_pipeline_performance.yaml | 14 + .../media_generation_performance.yaml | 14 + .../benchmarks/task_output_parity.yaml | 10 + tools/benchmark_qualification/catalog.py | 30 +- tools/benchmark_qualification/performance.py | 60 +++- tools/benchmark_qualification/runtime.py | 23 +- tools/tests/test_model_benchmark.py | 6 +- 39 files changed, 1524 insertions(+), 124 deletions(-) create mode 100644 apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py create mode 100644 families/bark/tests/benchmark/bark-large.yaml create mode 100644 families/bark/tests/benchmark/bark-small.yaml create mode 100644 families/flux/tests/benchmark/flux-2-dev-fp8.yaml create mode 100644 families/flux/tests/benchmark/flux-2-dev.yaml create mode 100644 families/flux/tests/benchmark/flux-schnell.yaml create mode 100644 families/gemma/tests/benchmark/gemma-3-1b.yaml create mode 100644 families/gemma/tests/benchmark/gemma-3-270m.yaml create mode 100644 families/gemma/tests/benchmark/gemma-3-4b.yaml create mode 100644 families/lance/tests/benchmark/lance-3b-x2t-image.yaml create mode 100644 families/lance/tests/benchmark/prepare_environment.py create mode 100644 families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml create mode 100644 families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml create mode 100644 families/personaplex/tests/benchmark/personaplex-7b.yaml create mode 100644 families/personaplex/tests/benchmark/prepare_environment.py create mode 100644 families/pixart/tests/benchmark/pixart-sigma-1024.yaml create mode 100644 families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml create mode 100644 families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml create mode 100644 families/qwen_image/tests/benchmark/qwen-image-2512.yaml create mode 100644 families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml create mode 100644 families/qwen_image/tests/benchmark/qwen-image.yaml create mode 100644 families/sana_wm/tests/benchmark/prepare_environment.py create mode 100644 families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml create mode 100644 families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml create mode 100644 families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml create mode 100644 families/z_image/tests/benchmark/z-image-turbo.yaml create mode 100644 tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/media_generation_performance.yaml create mode 100644 tools/benchmark_qualification/benchmarks/task_output_parity.yaml diff --git a/apps/benchmark/performance/baselines/sana_wm_reference.py b/apps/benchmark/performance/baselines/sana_wm_reference.py index 72d729e2f2..a11f89a27e 100755 --- a/apps/benchmark/performance/baselines/sana_wm_reference.py +++ b/apps/benchmark/performance/baselines/sana_wm_reference.py @@ -57,6 +57,26 @@ def media_summary(video: Any) -> dict[str, int | str]: } +def materialize_media(video: Any, output: Path) -> dict[str, Any]: + import numpy as np + from PIL import Image + + frames = np.asarray(video) + summary: dict[str, Any] = media_summary(frames) + indices = sorted({0, len(frames) // 2, len(frames) - 1}) + directory = output.with_suffix(".media") + directory.mkdir(parents=True, exist_ok=True) + artifacts = [] + for index in indices: + values = frames[index] + scale = 255.0 if np.issubdtype(values.dtype, np.floating) else 1.0 + path = directory / f"{index:06d}.png" + Image.fromarray(np.clip(values * scale, 0, 255).astype(np.uint8)).convert("RGB").save(path) + artifacts.append(str(path.resolve())) + summary.update(artifact_indices=indices, frame_artifacts=artifacts) + return summary + + def _official_module(reference_repo: Path) -> Any: entrypoint = reference_repo / "inference_video_scripts/wm/inference_sana_wm.py" if not entrypoint.is_file(): @@ -162,7 +182,7 @@ def invoke() -> Any: video = official.apply_overlay(video, result["c2w"]) arguments.output.write_text( json.dumps( - {"samples_ms": samples, "output_summary": media_summary(video)}, + {"samples_ms": samples, "output_summary": materialize_media(video, arguments.output)}, indent=2, sort_keys=True, ) diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index 7549ea1084..a4ba428788 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -67,6 +67,7 @@ ) ADAPTERS = ( "hf-diffusers", + "hf-diffusers-modular", "hf-chat-asr", "hf-qwen3-omni", "hf-transformers-asr", @@ -124,7 +125,9 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--model", required=True) parser.add_argument("--revision") parser.add_argument("--manifest", required=True, type=Path) - parser.add_argument("--selected-task", help="Semantic Task selected independently of bundle identity") + parser.add_argument( + "--selected-task", help="Semantic Task selected independently of bundle identity" + ) parser.add_argument("--request-json", required=True) parser.add_argument("--adapter-options-json", default="{}") parser.add_argument("--timing-contract-json", default="{}") @@ -138,7 +141,9 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--warmup", required=True, type=int) parser.add_argument("--iterations", required=True, type=int) parser.add_argument("--case-name", required=True) - parser.add_argument("--testcase-name", help="Selected manifest testcase, not the performance entry ID") + parser.add_argument( + "--testcase-name", help="Selected manifest testcase, not the performance entry ID" + ) parser.add_argument("--output", required=True, type=Path) return parser @@ -342,10 +347,16 @@ def _regression_values_summary(value: Any) -> dict[str, Any]: raise ValueError("deterministic regression requires one batch with a nonempty target axis") if not summary["finite"]: raise ValueError("deterministic regression target values must be finite") - return {"kind": "regression_values", "target_count": shape[1], - "regression_targets": shape[1], "parameter_elements": 0, - "values": value[0].detach().float().cpu().tolist(), "axes": ["target"], - "target_names": [], "target_units": []} + return { + "kind": "regression_values", + "target_count": shape[1], + "regression_targets": shape[1], + "parameter_elements": 0, + "values": value[0].detach().float().cpu().tolist(), + "axes": ["target"], + "target_names": [], + "target_units": [], + } def _regression_summary( @@ -365,7 +376,9 @@ def _regression_summary( "negative_binomial": {"total_count", "logits"}, }[distribution] if len(names) != len(required) or set(names) != required: - raise ValueError("regression reference parameter names do not match the declared distribution") + raise ValueError( + "regression reference parameter names do not match the declared distribution" + ) parameters = [] targets = None for name, value in zip(names, output): @@ -458,7 +471,11 @@ def invoke() -> Mapping[str, Any]: with torch.inference_mode(): audio, length = model.do_tts(transcript=prompt, language="en", use_cfg=True) sample_count = int(length.item()) if length.numel() else int(audio.numel()) - return {"audio_samples": sample_count, "sample_rate": 22_050} + return { + "audio_samples": sample_count, + "sample_rate": 22_050, + "_audio_f32": audio.detach().float().cpu().reshape(-1)[:sample_count].numpy(), + } else: from transformers import AutoProcessor, BarkModel @@ -483,6 +500,7 @@ def invoke() -> Mapping[str, Any]: return { "audio_samples": int(audio.numel()), "sample_rate": int(model.generation_config.sample_rate), + "_audio_f32": audio.detach().float().cpu().reshape(-1).numpy(), } if arguments.family == "magpie_tts": @@ -910,12 +928,16 @@ def _load_embedding( request: Mapping[str, Any], _options: Mapping[str, Any], ) -> Session: - configured = _json_object(getattr(arguments, "timing_contract_json", "{}"), "--timing-contract-json") + configured = _json_object( + getattr(arguments, "timing_contract_json", "{}"), "--timing-contract-json" + ) if not configured: raise ValueError("embedding reference requires explicit --timing-contract-json") declared_timing = timing_contract(runner="task-reference", declared=configured) if declared_timing["asset_loading_included"]: - raise ValueError("embedding reference preloads its assets; asset_loading_included must be false") + raise ValueError( + "embedding reference preloads its assets; asset_loading_included must be false" + ) prompts = _batch_prompt(request) if len(prompts) != 1: raise ValueError("embedding reference requires exactly one text input") @@ -929,9 +951,7 @@ def _load_embedding( .eval() .to(device) ) - compile_evidence = ( - _compile_forward(model) if arguments.mode == "torch-compile" else None - ) + compile_evidence = _compile_forward(model) if arguments.mode == "torch-compile" else None prompt = prompts[0] def prepare_inputs() -> Mapping[str, Any]: @@ -1186,7 +1206,9 @@ def _load_diffusers( if steps > 0: values.update({"num_inference_steps": steps, "step": steps}) manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) - height = int(request.get("height", request.get("video_height", manifest.get("image_height", 0)))) + height = int( + request.get("height", request.get("video_height", manifest.get("image_height", 0))) + ) width = int(request.get("width", request.get("video_width", manifest.get("image_width", 0)))) if height > 0: values["height"] = height @@ -1288,7 +1310,7 @@ def invoke() -> Mapping[str, Any]: if media is None: media = getattr(result, "frames", None) media_type = str(request.get("media_type", "image")) - return _media_summary(media, media_type) + return {**_media_summary(media, media_type), "_media": media} return Session( invoke, @@ -1298,6 +1320,69 @@ def invoke() -> Mapping[str, Any]: ) +def _load_modular_diffusers( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + """Run a ModularPipeline whose workflow is declared by the family profile.""" + import torch + from diffusers import ModularPipeline + + workflow = str(options.get("workflow", "")) + output_name = str(options.get("output", "videos")) + if not workflow or not output_name: + raise ValueError("modular Diffusers requires adapter_options.workflow and output") + load_options = { + "workflow": workflow, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + load_options["revision"] = arguments.revision + pipeline = ModularPipeline.from_pretrained(arguments.model, **load_options) + pipeline.load_components( + dtype=_torch_dtype(torch, arguments.precision), + pretrained_model_name_or_path=arguments.model, + local_files_only=arguments.local_files_only, + ) + pipeline = pipeline.to("cuda") + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + seed = int(request.get("seed", 42)) + generator = torch.Generator().manual_seed(seed) + values: dict[str, Any] = { + "prompt": str(request.get("prompt", "")), + "height": int(request.get("height", manifest.get("image_height", 0))), + "width": int(request.get("width", manifest.get("image_width", 0))), + "num_inference_steps": int(request.get("num_steps", 0)), + "generator": generator, + "output": output_name, + "output_type": "np", + } + frames = int(request.get("video_num_frames", manifest.get("video_num_frames", 1))) + if frames > 1: + values["num_frames"] = frames + for source, target in ( + ("negative_prompt", "negative_prompt"), + ("guidance_scale", "guidance_scale"), + ("cfg_scale", "cfg_scale"), + ): + if source in request: + values[target] = request[source] + + def invoke() -> Mapping[str, Any]: + generator.manual_seed(seed) + media = pipeline(**values) + media_type = "video" if frames > 1 else "image" + return {**_media_summary(media, media_type), "_media": media} + + return Session( + invoke, + "diffusers-modular", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + ) + + def _media_count(media: Any, media_type: str) -> int: """Count image batches or video frames without coercing arrays to bool.""" if media is None: @@ -1358,6 +1443,58 @@ def _media_summary(media: Any, media_type: str) -> dict[str, Any]: return summary +def _media_items(media: Any, media_type: str) -> list[Any]: + """Return image items, or frames from the first generated video.""" + import numpy as np + + if media_type == "video": + if isinstance(media, np.ndarray): + values = media[0] if media.ndim == 5 else media + else: + values = media[0] if isinstance(media, (list, tuple)) and media else media + else: + values = media + if isinstance(values, np.ndarray): + if values.ndim == 3: + return [values] + return [values[index] for index in range(len(values))] + if isinstance(values, (list, tuple)): + return list(values) + return [values] + + +def _write_media_artifacts(summary: dict[str, Any], output: Path) -> None: + """Materialize deterministic sample frames after reference timing completes.""" + import numpy as np + from PIL import Image + + media = summary.pop("_media") + items = _media_items(media, str(summary.get("media_type", "image"))) + declared = int(summary.get("media_count", 0)) + if declared != len(items) or declared < 1: + raise RuntimeError("reference media count differs from its materialized output") + indices = sorted({0, declared // 2, declared - 1}) + directory = output.with_suffix(".media").resolve() + directory.mkdir(parents=True, exist_ok=True) + artifacts = [] + for index in indices: + item = items[index] + values = np.asarray(item) + scale = 255.0 if np.issubdtype(values.dtype, np.floating) else 1.0 + image = ( + item + if isinstance(item, Image.Image) + else Image.fromarray(np.clip(values * scale, 0, 255).astype(np.uint8)) + ) + path = directory / f"{index:06d}.png" + image.convert("RGB").save(path) + artifacts.append(str(path)) + summary["artifact_indices"] = indices + summary["frame_artifacts" if summary.get("media_type") == "video" else "image_artifacts"] = ( + artifacts + ) + + def _numeric_values(request: Mapping[str, Any], key: str) -> list[float]: raw = request.get(key) if not isinstance(raw, list) or not raw: @@ -1428,8 +1565,11 @@ def _load_timeseries( compile_evidence = _compile_forward(model.model) raw = _numeric_values(request, "past_values") observed = _observed_values(request, len(raw)) - context = torch.tensor([value if mask > 0 else float("nan") - for value, mask in zip(raw, observed)], dtype=dtype, device=device) + context = torch.tensor( + [value if mask > 0 else float("nan") for value, mask in zip(raw, observed)], + dtype=dtype, + device=device, + ) def invoke() -> Mapping[str, Any]: with torch.inference_mode(): @@ -1540,8 +1680,9 @@ def invoke() -> Mapping[str, Any]: ) output = getattr(outputs, output_name) if task_id == "series_to_regression_distribution": - return _regression_summary(output, config.distribution_output, - tuple(model.distribution_output.args_dim)) + return _regression_summary( + output, config.distribution_output, tuple(model.distribution_output.args_dim) + ) if task_id == "series_to_regression_values": return _regression_values_summary(output) if isinstance(output, (tuple, list)): @@ -2157,8 +2298,6 @@ def _write_geometry_artifacts(summary: dict[str, Any], output: Path) -> None: ) - - def _load_personaplex( arguments: argparse.Namespace, request: Mapping[str, Any], @@ -2222,6 +2361,21 @@ def invoke() -> Mapping[str, Any]: other_mimi.reset_streaming() generator.reset_streaming() generated_frames = 0 + decoded_chunks = [] + + def summary() -> Mapping[str, Any]: + audio = ( + torch.cat(decoded_chunks, dim=-1).detach().float().cpu().reshape(-1).numpy() + if decoded_chunks + else torch.empty(0, dtype=torch.float32).numpy() + ) + return { + "audio_frames": generated_frames, + "audio_samples": int(audio.size), + "sample_rate": mimi.sample_rate, + "_audio_f32": audio, + } + with torch.inference_mode(): for encoded in encode_from_sphn( mimi, @@ -2232,20 +2386,12 @@ def invoke() -> Mapping[str, Any]: tokens = generator.step(encoded[:, :, index : index + 1]) if tokens is None: continue - mimi.decode(tokens[:, 1:9]) + decoded_chunks.append(mimi.decode(tokens[:, 1:9])) other_mimi.decode(tokens[:, 1:9]) generated_frames += 1 if generated_frames >= max_frames: - return { - "audio_frames": generated_frames, - "audio_samples": generated_frames * frame_size, - "sample_rate": mimi.sample_rate, - } - return { - "audio_frames": generated_frames, - "audio_samples": generated_frames * frame_size, - "sample_rate": mimi.sample_rate, - } + return summary() + return summary() return Session( invoke, @@ -2351,6 +2497,7 @@ def invoke() -> Mapping[str, Any]: str, Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session] ] = { "hf-diffusers": _load_diffusers, + "hf-diffusers-modular": _load_modular_diffusers, "hf-chat-asr": _load_asr, "hf-qwen3-omni": _load_qwen3_omni, "hf-transformers-asr": _load_asr, @@ -2709,12 +2856,19 @@ def _run_sana_wm( if testcase_name is not None: manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) matches = [ - value for value in manifest.get("testcases", []) + value + for value in manifest.get("testcases", []) if isinstance(value, Mapping) and value.get("name") == testcase_name ] if len(matches) != 1: raise ValueError(f"SANA-WM requires exactly one testcase named {testcase_name!r}") - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay"): + for name in ( + "translation_speed", + "rotation_speed_deg", + "fps", + "flow_shift", + "no_action_overlay", + ): if name in matches[0]: options.setdefault(name, matches[0][name]) reference_repo = str(options.get("reference_repo", "")) @@ -2737,7 +2891,9 @@ def model_input(value: object) -> Path: action = request.get("action") if not isinstance(action, str) or not action: raise ValueError("SANA-WM request requires a non-empty action") - intrinsics = model_input(options.get("intrinsics", "assets/demo_0_intrinsics.npy")) + intrinsics = model_input( + options.get("intrinsics_path", options.get("intrinsics", "assets/demo_0_intrinsics.npy")) + ) for label, path in (("image", image), ("intrinsics", intrinsics)): if not path.is_file(): raise FileNotFoundError(f"SANA-WM {label} input does not exist: {path}") @@ -2768,9 +2924,17 @@ def model_input(value: object) -> Path: "--intrinsics", str(intrinsics), "--translation_speed", - str(request["translation_speed"] if "translation_speed" in request else options["translation_speed"]), + str( + request["translation_speed"] + if "translation_speed" in request + else options["translation_speed"] + ), "--rotation_speed_deg", - str(request["rotation_speed_deg"] if "rotation_speed_deg" in request else options["rotation_speed_deg"]), + str( + request["rotation_speed_deg"] + if "rotation_speed_deg" in request + else options["rotation_speed_deg"] + ), "--num_frames", str(num_frames), "--fps", @@ -2792,7 +2956,11 @@ def model_input(value: object) -> Path: "--output", str(output), ] - if request["no_action_overlay"] if "no_action_overlay" in request else options["no_action_overlay"]: + if ( + request["no_action_overlay"] + if "no_action_overlay" in request + else options["no_action_overlay"] + ): command.append("--no_action_overlay") completed = subprocess.run( command, @@ -2807,6 +2975,19 @@ def model_input(value: object) -> Path: f"rc={completed.returncode}: {completed.stderr[-4000:]}" ) payload = json.loads(output.read_text(encoding="utf-8")) + summary = dict(payload.get("output_summary", {})) + source_artifacts = summary.get("frame_artifacts", []) + if source_artifacts: + destination = arguments.output.with_suffix(".media").resolve() + destination.mkdir(parents=True, exist_ok=True) + copied = [] + for source_value in source_artifacts: + source = Path(str(source_value)) + target = destination / source.name + shutil.copyfile(source, target) + copied.append(str(target)) + summary["frame_artifacts"] = copied + payload["output_summary"] = summary samples = [float(value) for value in payload.get("samples_ms", [])] if len(samples) != arguments.iterations: raise RuntimeError( @@ -2857,9 +3038,7 @@ def run(arguments: argparse.Namespace) -> int: if arguments.adapter == "hf-transformers-embedding": supported_modes.add("torch-compile") if arguments.mode not in supported_modes: - raise ValueError( - f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}" - ) + raise ValueError(f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}") request = flatten_config(_json_object(arguments.request_json, "--request-json")) options = _json_object(arguments.adapter_options_json, "--adapter-options-json") configured_timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") @@ -2867,7 +3046,9 @@ def run(arguments: argparse.Namespace) -> int: expected_timing = None if configured_timing: if set(configured_timing) != set(fields): - raise ValueError("--timing-contract-json must declare exactly the three reference timing fields") + raise ValueError( + "--timing-contract-json must declare exactly the three reference timing fields" + ) declared = timing_contract(runner="task-reference", declared=configured_timing) expected_timing = {name: declared[name] for name in fields} load_started = time.perf_counter() @@ -2924,6 +3105,8 @@ def run(arguments: argparse.Namespace) -> int: f"actual={actual_timing}, declared={expected_timing}" ) samples, output_summary = _measure(session, arguments.warmup, arguments.iterations) + if "_media" in output_summary: + _write_media_artifacts(output_summary, arguments.output) if "_geometry_arrays" in output_summary: _write_geometry_artifacts(output_summary, arguments.output) disparity = output_summary.pop("_disparity_f32", None) @@ -2935,6 +3118,7 @@ def run(arguments: argparse.Namespace) -> int: audio = output_summary.pop("_audio_f32", None) if audio is not None: import soundfile as sf + artifact_path = arguments.output.with_suffix(".audio.wav").resolve() artifact_path.parent.mkdir(parents=True, exist_ok=True) sf.write(artifact_path, audio, output_summary["sample_rate"], subtype="FLOAT") diff --git a/apps/benchmark/trtmc_benchmark/artifact_metrics.py b/apps/benchmark/trtmc_benchmark/artifact_metrics.py index a48d290cac..87772d5590 100644 --- a/apps/benchmark/trtmc_benchmark/artifact_metrics.py +++ b/apps/benchmark/trtmc_benchmark/artifact_metrics.py @@ -10,6 +10,170 @@ from typing import Any, Mapping +def generated_media_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare deterministic generated image/video artifacts without family semantics.""" + import numpy as np + from PIL import Image + + actual, actual_count, actual_indices = _media_artifacts(candidate, "candidate") + expected, expected_count, expected_indices = _media_artifacts(reference, "reference") + if actual_count != expected_count: + raise ValueError("candidate and reference media counts differ") + common_indices = sorted(set(actual_indices) & set(expected_indices)) + required_indices = sorted({0, actual_count // 2, actual_count - 1}) + if not set(required_indices).issubset(common_indices): + raise ValueError("candidate and reference media do not retain the required frames") + actual_by_index = dict(zip(actual_indices, actual, strict=True)) + expected_by_index = dict(zip(expected_indices, expected, strict=True)) + psnr_values = [] + ssim_values = [] + for index in required_indices: + left_path, right_path = actual_by_index[index], expected_by_index[index] + left = np.asarray(Image.open(left_path).convert("RGB"), dtype=np.float64) + right = np.asarray(Image.open(right_path).convert("RGB"), dtype=np.float64) + if left.shape != right.shape: + raise ValueError("candidate and reference media dimensions differ") + delta = left - right + mse = float(np.mean(delta * delta)) + psnr_values.append(100.0 if mse == 0.0 else 20.0 * math.log10(255.0 / math.sqrt(mse))) + left_mean, right_mean = float(left.mean()), float(right.mean()) + left_var, right_var = float(left.var()), float(right.var()) + covariance = float(np.mean((left - left_mean) * (right - right_mean))) + c1, c2 = (0.01 * 255.0) ** 2, (0.03 * 255.0) ** 2 + numerator = (2.0 * left_mean * right_mean + c1) * (2.0 * covariance + c2) + denominator = (left_mean**2 + right_mean**2 + c1) * (left_var + right_var + c2) + ssim_values.append(numerator / denominator) + return { + "media_count": float(actual_count), + "compared_frames": float(len(required_indices)), + "min_psnr": min(psnr_values), + "min_ssim": min(ssim_values), + } + + +def generated_media_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: + minimum_psnr = _finite_threshold(thresholds, "min_psnr") + minimum_ssim = _finite_threshold(thresholds, "min_ssim") + return metrics["min_psnr"] >= minimum_psnr and metrics["min_ssim"] >= minimum_ssim + + +def generated_audio_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare complete generated waveforms using stable signal-level metrics.""" + import numpy as np + + actual, actual_rate = _read_wav( + _artifact_path(candidate.get("audio_artifact"), "candidate audio"), np + ) + expected, expected_rate = _read_wav( + _artifact_path(reference.get("audio_artifact"), "reference audio"), np + ) + if actual_rate <= 0 or expected_rate <= 0 or actual.size == 0 or expected.size == 0: + raise ValueError("candidate and reference audio must be non-empty") + actual = actual.astype(np.float64, copy=False) + expected = expected.astype(np.float64, copy=False) + if not np.isfinite(actual).all() or not np.isfinite(expected).all(): + raise ValueError("candidate and reference audio must be finite") + duration_ratio = (actual.size / actual_rate) / (expected.size / expected_rate) + actual_rms = float(np.sqrt(np.mean(actual * actual))) + expected_rms = float(np.sqrt(np.mean(expected * expected))) + rms_ratio = actual_rms / max(expected_rms, 1.0e-12) + length = min(actual.size, expected.size) + # Compare a bounded aligned prefix; generation duration is gated separately. + fft_size = min(length, 262144) + window = np.hanning(fft_size) + left = np.log1p(np.abs(np.fft.rfft(actual[:fft_size] * window))) + right = np.log1p(np.abs(np.fft.rfft(expected[:fft_size] * window))) + log_spectral_distance = float(np.sqrt(np.mean((left - right) ** 2))) + return { + "duration_ratio": float(duration_ratio), + "rms_ratio": float(rms_ratio), + "log_spectral_distance": log_spectral_distance, + } + + +def generated_audio_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: + return ( + metrics["duration_ratio"] >= _finite_threshold(thresholds, "min_duration_ratio") + and metrics["duration_ratio"] <= _finite_threshold(thresholds, "max_duration_ratio") + and metrics["rms_ratio"] >= _finite_threshold(thresholds, "min_rms_ratio") + and metrics["rms_ratio"] <= _finite_threshold(thresholds, "max_rms_ratio") + and metrics["log_spectral_distance"] + <= _finite_threshold(thresholds, "max_log_spectral_distance") + ) + + +def _media_artifacts(summary: Mapping[str, Any], label: str) -> tuple[list[Path], int, list[int]]: + values = summary.get("frame_artifacts", summary.get("image_artifacts")) + if not isinstance(values, list) or not values: + raise ValueError(f"{label} media artifacts are missing") + paths = [_artifact_path(value, f"{label} media artifact") for value in values] + declared_value = summary.get( + "media_count", summary.get("num_frames", summary.get("generated_frames")) + ) + declared = _integer(declared_value, f"{label} media_count", minimum=1) + configured_indices = summary.get("artifact_indices") + indices = list(range(len(paths))) if configured_indices is None else configured_indices + if ( + not isinstance(indices, list) + or len(indices) != len(paths) + or any(isinstance(value, bool) or not isinstance(value, int) for value in indices) + or sorted(set(indices)) != indices + or indices[0] < 0 + or indices[-1] >= declared + ): + raise ValueError(f"{label} media artifact indices are invalid") + return paths, declared, indices + + +def _read_wav(path: Path, np): + import struct + + payload = path.read_bytes() + if len(payload) < 12 or payload[:4] != b"RIFF" or payload[8:12] != b"WAVE": + raise ValueError("audio artifact must be a RIFF/WAVE file") + chunks = {} + offset = 12 + while offset + 8 <= len(payload): + name = payload[offset : offset + 4] + size = int.from_bytes(payload[offset + 4 : offset + 8], "little") + start, end = offset + 8, offset + 8 + size + if end > len(payload): + raise ValueError("audio artifact contains a truncated WAV chunk") + if name in {b"fmt ", b"data"}: + chunks[name] = payload[start:end] + offset = end + (size & 1) + if len(chunks.get(b"fmt ", b"")) < 16 or b"data" not in chunks: + raise ValueError("audio artifact has no complete format and data chunks") + kind, channels, rate, _, alignment, bits = struct.unpack_from(" float: + value = thresholds.get(name) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ValueError(f"threshold {name} must be finite") + return float(value) + + def metric_geometry_metrics( candidate: Mapping[str, Any], reference: Mapping[str, Any] ) -> dict[str, float]: @@ -36,8 +200,7 @@ def metric_geometry_metrics( ref_points_valid = ref_points[common].astype(np.float64) point_delta = actual_points_valid - ref_points_valid cosine_denominator = np.maximum( - np.linalg.norm(actual_points_valid, axis=-1) - * np.linalg.norm(ref_points_valid, axis=-1), + np.linalg.norm(actual_points_valid, axis=-1) * np.linalg.norm(ref_points_valid, axis=-1), 1.0e-12, ) ref_intrinsics64 = ref_intrinsics.astype(np.float64) @@ -49,18 +212,13 @@ def metric_geometry_metrics( np.mean(np.abs(depth_delta) / np.maximum(np.abs(ref_depth_valid), 1.0e-12)) ), "depth_rel_l2": float( - np.linalg.norm(depth_delta) - / max(float(np.linalg.norm(ref_depth_valid)), 1.0e-12) + np.linalg.norm(depth_delta) / max(float(np.linalg.norm(ref_depth_valid)), 1.0e-12) ), "points_rel_l2": float( - np.linalg.norm(point_delta) - / max(float(np.linalg.norm(ref_points_valid)), 1.0e-12) + np.linalg.norm(point_delta) / max(float(np.linalg.norm(ref_points_valid)), 1.0e-12) ), "points_cosine": float( - np.mean( - np.sum(actual_points_valid * ref_points_valid, axis=-1) - / cosine_denominator - ) + np.mean(np.sum(actual_points_valid * ref_points_valid, axis=-1) / cosine_denominator) ), "intrinsics_max_relative_error": float( np.max(np.abs(intrinsics_delta[nonzero]) / np.abs(ref_intrinsics64[nonzero])) @@ -74,9 +232,7 @@ def metric_geometry_metrics( return metrics -def metric_geometry_passes( - metrics: Mapping[str, float], thresholds: Mapping[str, Any] -) -> bool: +def metric_geometry_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: """Apply the public metric-geometry threshold vocabulary.""" required = { "mask_iou", @@ -123,9 +279,7 @@ def robot_action_metrics( return metrics -def robot_action_passes( - metrics: Mapping[str, float], thresholds: Mapping[str, Any] -) -> bool: +def robot_action_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: """Apply the public robot-action threshold vocabulary.""" required = { "action_max_abs_error", diff --git a/apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py b/apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py new file mode 100644 index 0000000000..31cc7cbda4 --- /dev/null +++ b/apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +import wave + +import numpy as np +from PIL import Image + +from trtmc_benchmark.artifact_metrics import ( + generated_audio_metrics, + generated_audio_passes, + generated_media_metrics, + generated_media_passes, +) + + +def test_generated_media_compares_retained_video_frames(tmp_path: Path) -> None: + left = tmp_path / "left.png" + right = tmp_path / "right.png" + pixels = np.full((8, 8, 3), 96, dtype=np.uint8) + Image.fromarray(pixels).save(left) + Image.fromarray(pixels).save(right) + candidate = { + "media_count": 5, + "artifact_indices": [0, 2, 4], + "frame_artifacts": [str(left), str(left), str(left)], + } + reference = { + "media_count": 5, + "artifact_indices": [0, 2, 4], + "frame_artifacts": [str(right), str(right), str(right)], + } + + metrics = generated_media_metrics(candidate, reference) + + assert metrics["media_count"] == 5 + assert metrics["min_psnr"] == 100.0 + assert generated_media_passes(metrics, {"min_psnr": 5.0, "min_ssim": 0.1}) + + +def test_generated_audio_compares_complete_waveforms(tmp_path: Path) -> None: + sample_rate = 16_000 + time = np.arange(sample_rate, dtype=np.float32) / sample_rate + samples = np.sin(2.0 * np.pi * 440.0 * time).astype(np.float32) + candidate = tmp_path / "candidate.wav" + reference = tmp_path / "reference.wav" + pcm = np.rint(samples * 32767.0).astype(" None: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "Lance" + if (destination / "inference_lance.py").is_file(): + return + if destination.exists(): + raise RuntimeError(f"incomplete Lance reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="Lance.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", REPOSITORY], check=True + ) + subprocess.run( + ["git", "-C", str(temporary), "fetch", "--quiet", "--depth=1", "origin", REVISION], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +if __name__ == "__main__": + main() diff --git a/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml b/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml new file mode 100644 index 0000000000..92265c9293 --- /dev/null +++ b/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: magpie-tts-357m +candidate: {family: magpie_tts, checkpoint: nvidia/magpie_tts_multilingual_357m, revision: 34d7e40da85cabc97f92198889b65cea27bc7fd1, task: audio_generation, precision: fp32, bundle: magpie-tts-357m.bundle, build: {max_sequence_length: 512}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-audio-parity + benchmark: task_output_parity + operation: generate_audio + request: &request {prompt: Please say this sentence clearly., max_new_tokens: 260, seed: 42} + reference: &reference {adapter: nemo-tts, mode: pytorch-eager, precision: fp32, output_contract: audio-artifact-parity, adapter_options: {speaker_encoder_revision: e9124b5364a2c3e9b4f78da429a33cbca8f8c22b}} + gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} +performance: + - {name: generate-audio, benchmark: audio_generation_pipeline_performance, operation: generate_audio, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: nemo_reference, output_contract: audio-shape}} diff --git a/families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml b/families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml new file mode 100644 index 0000000000..2a916c72d5 --- /dev/null +++ b/families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: minimax-h3-768p +candidate: + family: minimax_h3 + checkpoint: MiniMaxAI/MiniMax-H3 + revision: 48d93ede732756e404a3b1b2f3b3a9b5a22f6cfc + task: image_generation + precision: bf16 + bundle: minimax-h3-768p.bundle + build: {image_height: 768, image_width: 1344, video_num_frames: 124} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt_path: ../prompts/t2va-example-1.json, media_type: video, seed: 0, num_steps: 50, height: 768, width: 1344, video_num_frames: 124} + reference: &reference {adapter: hf-diffusers-modular, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {workflow: t2va, output: videos}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/personaplex/tests/benchmark/personaplex-7b.yaml b/families/personaplex/tests/benchmark/personaplex-7b.yaml new file mode 100644 index 0000000000..ae07c3a6ff --- /dev/null +++ b/families/personaplex/tests/benchmark/personaplex-7b.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: personaplex-7b +candidate: {family: personaplex, checkpoint: nvidia/personaplex-7b-v1, revision: fdaf4090a61cb315c138a1faee287ffd6c716309, task: speech_to_speech, precision: fp16, bundle: personaplex-7b.bundle, build: {max_sequence_length: 512, fp32_layers: [0, 1]}} +reference_environment: + requirements: ../../requirements.txt + paths: {personaplex_repo: trtmc-reference/personaplex} +accuracy: + - name: generated-audio-parity + benchmark: task_output_parity + operation: speak + request: &request {audio_path: ../data/Recording.wav, max_new_tokens: 300} + reference: &reference {adapter: pytorch-personaplex, mode: pytorch-eager, precision: bf16, output_contract: audio-artifact-parity} + gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} +performance: + - {name: speak, benchmark: audio_generation_pipeline_performance, operation: speak, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: torch_reference, output_contract: audio-shape}} diff --git a/families/personaplex/tests/benchmark/prepare_environment.py b/families/personaplex/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..9e043ac699 --- /dev/null +++ b/families/personaplex/tests/benchmark/prepare_environment.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned family-owned PersonaPlex reference checkout.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +REPOSITORY = "https://github.com/NVIDIA/personaplex.git" +REVISION = "3428dfd95309a7f3c84fd93259ded0f810d1ff91" + + +def main() -> None: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "personaplex" + if (destination / "moshi").is_dir(): + return + if destination.exists(): + raise RuntimeError(f"incomplete PersonaPlex reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="personaplex.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", REPOSITORY], check=True + ) + subprocess.run( + ["git", "-C", str(temporary), "fetch", "--quiet", "--depth=1", "origin", REVISION], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +if __name__ == "__main__": + main() diff --git a/families/pixart/tests/benchmark/pixart-sigma-1024.yaml b/families/pixart/tests/benchmark/pixart-sigma-1024.yaml new file mode 100644 index 0000000000..1e9b265ac0 --- /dev/null +++ b/families/pixart/tests/benchmark/pixart-sigma-1024.yaml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: pixart-sigma-1024 +candidate: + family: pixart + checkpoint: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS + task: image_generation + precision: fp16 + bundle: pixart-sigma-1024.bundle + build: {fp32_layers: [0], image_height: 1024, image_width: 1024} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 20, height: 1024, width: 1024} + reference: &reference + adapter: hf-diffusers + mode: hf-eager + precision: fp16 + output_contract: media-artifact-parity + adapter_options: {component_precision_contract: pixart_fp16_dit_fp32_t5} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - name: generate-image + benchmark: media_generation_performance + operation: generate_image + request: *request + measurement: {warmup: 5, iterations: 10} + reference: + <<: *reference + runner: task-reference + reference_backend: hf_diffusers + output_contract: media-shape diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml new file mode 100644 index 0000000000..c9a9d2f944 --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-0.6b-topp +candidate: + family: qwen + checkpoint: Qwen/Qwen3-0.6B + task: text_generation + precision: fp16 + bundle: qwen3-0.6b-topp-fp16.bundle + build: {max_sequence_length: 257} +accuracy: + - name: mmlu-sampling + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: {precision: fp32} + request: &sampling + {max_new_tokens: 20, temperature: 0.7, top_k: 50, top_p: 0.9, min_p: 0.0, seed: 42, repetition_penalty: 1.0, use_chat_template: false, enable_thinking: false, do_sample: true} + gate: {min_pass_rate: 0.9, allowed_failures: 1} +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *sampling + prompt: What is the capital of France? Answer in one word. + use_chat_template: true + measurement: {warmup: 5, iterations: 10} + reference: {runner: hf-transformers, mode: torch-compile, fallback: hf-eager, compile_scope: model.forward, precision: fp32, output_contract: exact-token-ids} diff --git a/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml b/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml new file mode 100644 index 0000000000..2484898d41 --- /dev/null +++ b/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen3-omni-30b-a3b-instruct +candidate: {family: qwen3_omni, checkpoint: Qwen/Qwen3-Omni-30B-A3B-Instruct, revision: 26291f793822fb6be9555850f06dfe95f2d7e695, task: text_generation, precision: bf16, bundle: qwen3-omni-30b-a3b-instruct.bundle, build: {max_sequence_length: 256}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-text-parity + benchmark: task_output_parity + operation: generate + request: &request {prompt: Please say hello from Qwen3 Omni in one short sentence., max_new_tokens: 16} + reference: &reference {adapter: hf-qwen3-omni, mode: hf-eager, precision: bf16, output_contract: normalized-text} + gate: {max_normalized_edit_distance: 0.15} +performance: + - {name: generate, benchmark: vision_language_performance, operation: generate, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_transformers, output_contract: exact-text}} diff --git a/families/qwen_image/tests/benchmark/qwen-image-2512.yaml b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml new file mode 100644 index 0000000000..cba81cfecf --- /dev/null +++ b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen-image-2512 +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-2512, task: image_generation, precision: bf16, bundle: qwen-image-2512.bundle, build: {image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - {name: generated-media-parity, benchmark: task_output_parity, operation: generate_image, request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024}, reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}}, gate: {min_psnr: 5.0, min_ssim: 0.1}} +performance: + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml b/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml new file mode 100644 index 0000000000..b284ae0eeb --- /dev/null +++ b/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen-image-edit-2511 +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-Edit-2511, task: image_edit, precision: bf16, bundle: qwen-image-edit-2511.bundle, build: {image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: Turn the image into a watercolor illustration while preserving the main subject., image_path: ../data/test_img.jpeg, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: edit-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_image/tests/benchmark/qwen-image.yaml b/families/qwen_image/tests/benchmark/qwen-image.yaml new file mode 100644 index 0000000000..d3c0354f55 --- /dev/null +++ b/families/qwen_image/tests/benchmark/qwen-image.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen-image +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image, task: image_generation, precision: bf16, bundle: qwen-image.bundle, build: {image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/sana_wm/tests/benchmark/prepare_environment.py b/families/sana_wm/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..46da0d6d6a --- /dev/null +++ b/families/sana_wm/tests/benchmark/prepare_environment.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare pinned SANA source and a local HF-compatible model directory.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +from huggingface_hub import snapshot_download + +REPOSITORY = "https://github.com/NVlabs/Sana.git" +REVISION = "59629fdf790850797cb657bad014fce432bd713d" +MODEL = "Efficient-Large-Model/SANA-WM_bidirectional" + + +def main() -> None: + parent = Path(sys.prefix) / "trtmc-reference" + source = parent / "Sana" + parent.mkdir(parents=True, exist_ok=True) + if not (source / "inference_video_scripts/wm/inference_sana_wm.py").is_file(): + if source.exists(): + raise RuntimeError(f"incomplete SANA reference: {source}") + temporary = Path(tempfile.mkdtemp(prefix="Sana.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", REPOSITORY], check=True + ) + subprocess.run( + ["git", "-C", str(temporary), "fetch", "--quiet", "--depth=1", "origin", REVISION], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(source) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + model = parent / "SANA-model" + if not model.exists(): + model.symlink_to(Path(snapshot_download(MODEL)).resolve(), target_is_directory=True) + + +if __name__ == "__main__": + main() diff --git a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml new file mode 100644 index 0000000000..1fa221f256 --- /dev/null +++ b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: sana-wm-bidirectional +candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} +reference_environment: + requirements: ../../requirements.txt + paths: {sana_repo: trtmc-reference/Sana, sana_model: trtmc-reference/SANA-model} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request + prompt_path: ../assets/demo_0.txt + image_path: ../assets/demo_0.png + action: w-80,jw-40,w-40,lw-60,w-100 + camera_intrinsics: [797.87866, 830.0503, 844.2675, 463.7225] + media_type: video + seed: 42 + num_steps: 60 + cfg_scale: 5.0 + height: 704 + width: 1280 + video_num_frames: 321 + translation_speed: 0.055 + rotation_speed_deg: 1.2 + fps: 16 + flow_shift: 9.8 + no_action_overlay: true + reference: &reference {adapter: upstream-sana-wm, mode: pytorch-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {intrinsics_path: ../assets/demo_0_intrinsics.npy}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: official_sana, output_contract: media-shape}} diff --git a/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml b/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml new file mode 100644 index 0000000000..2b406e15de --- /dev/null +++ b/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: wan22-ti2v-5b +candidate: {family: wan2_2_ti2v, checkpoint: Wan-AI/Wan2.2-TI2V-5B, revision: 921dbaf3f1674a56f47e83fb80a34bac8a8f203e, task: image_generation, precision: bf16, bundle: wan22-ti2v-5b.bundle, build: {image_height: 704, image_width: 1280, video_num_frames: 121}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage, media_type: video, seed: 42, num_steps: 50, guidance_scale: 5.0, height: 704, width: 1280, video_num_frames: 121} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {model_id: Wan-AI/Wan2.2-TI2V-5B-Diffusers, model_revision: b8fff7315c768468a5333511427288870b2e9635}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml b/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml new file mode 100644 index 0000000000..a8f969e0b1 --- /dev/null +++ b/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: wan21-t2v-1.3b +candidate: {family: wan_t2v, checkpoint: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, task: image_generation, precision: fp16, bundle: wan21-t2v-1.3b.bundle, build: {context_parallel_size: 1, fp32_layers: [24], image_height: 480, image_width: 832, video_num_frames: 17}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A cat sitting on a beach watching the sunset, media_type: video, seed: 42, num_steps: 30, height: 480, width: 832, video_num_frames: 17} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: fp32, output_contract: media-artifact-parity} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/z_image/tests/benchmark/z-image-turbo.yaml b/families/z_image/tests/benchmark/z-image-turbo.yaml new file mode 100644 index 0000000000..4e6664f62e --- /dev/null +++ b/families/z_image/tests/benchmark/z-image-turbo.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: z-image-turbo +candidate: {family: z_image, checkpoint: Tongyi-MAI/Z-Image-Turbo, task: image_generation, precision: fp16, bundle: z-image-turbo.bundle, build: {fp32_layers: [2, 3, 4, 7, 8], image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 9, guidance_scale: 0.0, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: fp16, output_contract: media-artifact-parity} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 7a6235fe87..82bf1b873d 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -10,12 +10,17 @@ from itertools import permutations import json import math +import os import random import re from pathlib import Path from typing import Any, Mapping, Sequence from trtmc_benchmark.artifact_metrics import ( + generated_audio_metrics, + generated_audio_passes, + generated_media_metrics, + generated_media_passes, metric_geometry_metrics, metric_geometry_passes, robot_action_metrics, @@ -29,6 +34,7 @@ benchmark_executable, prepare_bundle, reference_python, + reference_environment_paths, require_candidate, run_command, write_model_descriptor, @@ -40,9 +46,12 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, output = context.case_artifacts(case) output.mkdir(parents=True, exist_ok=True) definition = load_benchmark(context.repository, case) - dataset = resolve_dataset(definition, context, case.source) metric = definition.get("metric") metric_name = metric.get("name") if isinstance(metric, Mapping) else None + if metric_name == "task_output_parity": + result = _task_output_parity(case, context, output) + else: + dataset = resolve_dataset(definition, context, case.source) if metric_name == "exact_token_ids": result = _text_generation_parity(case, context, dataset, output) elif metric_name == "embedding_vector_parity": @@ -77,12 +86,177 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _metric_geometry_parity(case, context, dataset, output) elif metric_name == "robot_action_parity": result = _robot_action_parity(case, context, dataset, output) + elif metric_name == "task_output_parity": + pass else: raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") write_result(output, result) return result +_REFERENCE_OPTIONS = { + "upstream-lance": {"lance_repo": "reference_repo"}, + "upstream-sana-wm": {"sana_repo": "reference_repo", "sana_model": "model_dir"}, + "pytorch-personaplex": {"personaplex_repo": "official_repo"}, +} + + +def _task_output_parity( + case: QualificationCase, context: RuntimeContext, output: Path +) -> dict[str, Any]: + configured = case.values + request = configured.get("request", {}) + reference = configured.get("reference", {}) + gate = configured.get("gate", {}) + if not all(isinstance(value, Mapping) for value in (request, reference, gate)): + raise QualificationError("task-output request, reference, and gate must be objects") + request = _resolve_task_assets(case, request) + descriptor = write_model_descriptor(case, output, request, context=context) + descriptor_value = json.loads(descriptor.read_text(encoding="utf-8")) + adapter = str(reference.get("adapter", "")) + if not adapter: + raise QualificationError("task-output reference.adapter must be set") + adapter_options = dict(reference.get("adapter_options", {})) + family_root = case.source.parents[2].resolve() + for name, value in tuple(adapter_options.items()): + if not name.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if not path.is_absolute(): + path = (case.source.parent / path).resolve() + if not path.is_relative_to(family_root) or not path.is_file(): + raise QualificationError(f"task-output reference asset {name!r} is unavailable: {path}") + adapter_options[name] = str(path) + environment_paths = reference_environment_paths(case, context) + for source, target in _REFERENCE_OPTIONS.get(adapter, {}).items(): + if source not in environment_paths: + raise QualificationError(f"task-output reference path {source!r} is not configured") + adapter_options[target] = environment_paths[source] + reference_path = output / "reference-result.json" + command = [ + str(reference_python(case, context)), + str(context.repository / "apps/benchmark/performance/baselines/task_reference.py"), + "--adapter", + adapter, + "--family", + case.family, + "--operation", + str(configured["operation"]), + "--model", + str(descriptor_value["hf_id"]), + "--manifest", + str(descriptor), + "--testcase-name", + case.name, + "--request-json", + json.dumps(request), + "--adapter-options-json", + json.dumps(adapter_options), + "--precision", + str(reference.get("precision", case.candidate["precision"])), + "--mode", + str(reference.get("mode", "hf-eager")), + "--warmup", + "0", + "--iterations", + "1", + "--case-name", + case.name, + "--output", + str(reference_path), + ] + revision = reference.get("revision", descriptor_value.get("hf_revision")) + if revision: + command.extend(("--revision", str(revision))) + selected_task = case.candidate.get("selected_task") + if selected_task: + command.extend(("--selected-task", str(selected_task))) + if bool(case.candidate.get("trust_remote_code", False)): + command.append("--trust-remote-code") + if os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1": + command.append("--local-files-only") + completed = run_command(command, output, "reference", timeout=7200, verbose=context.verbose) + if completed.returncode != 0: + raise QualificationError(f"task-output Accuracy reference failed; see {output}") + try: + reference_result = json.loads(reference_path.read_text(encoding="utf-8")) + expected = reference_result["output_summary"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as error: + raise QualificationError("task-output Accuracy reference returned no output") from error + actual_values, bundle = _candidate_outputs( + case, + context, + output, + str(configured["operation"]), + [{"sample_id": case.name, "request": request}], + ) + actual = actual_values[0] + contract = str(reference.get("output_contract", "")) + if not isinstance(expected, Mapping): + raise QualificationError("task-output Accuracy reference returned an invalid output") + try: + if contract == "media-artifact-parity": + metrics = generated_media_metrics(actual, expected) + passed = generated_media_passes(metrics, gate) + elif contract == "audio-artifact-parity": + metrics = generated_audio_metrics(actual, expected) + passed = generated_audio_passes(metrics, gate) + elif contract == "normalized-text": + candidate_text = _normalized_answer(actual.get("text")) + reference_text = _normalized_answer(expected.get("text")) + distance = _normalized_edit_distance(candidate_text, reference_text) + metrics = {"normalized_edit_distance": distance} + passed = bool(candidate_text) and distance <= float( + gate["max_normalized_edit_distance"] + ) + else: + raise QualificationError(f"unsupported task-output contract {contract!r}") + except (KeyError, TypeError, ValueError) as error: + raise QualificationError(f"task-output Accuracy comparison failed: {error}") from error + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "reference": {"adapter": adapter, "model": reference_result.get("model")}, + "metrics": metrics, + "gate": dict(gate), + } + + +def _resolve_task_assets(case: QualificationCase, request: Mapping[str, Any]) -> dict[str, Any]: + resolved = dict(request) + family_root = case.source.parents[2].resolve() + for key, value in request.items(): + if not key.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if path.is_absolute(): + continue + path = (case.source.parent / path).resolve() + if not path.is_relative_to(family_root) or not path.is_file(): + raise QualificationError(f"task-output asset {key!r} is unavailable: {path}") + if key == "prompt_path": + payload = ( + json.loads(path.read_text(encoding="utf-8")) if path.suffix == ".json" else None + ) + prompt = ( + payload.get("prompt") + if isinstance(payload, Mapping) + else path.read_text(encoding="utf-8").strip() + ) + if not isinstance(prompt, str) or not prompt: + raise QualificationError(f"task-output prompt is unavailable: {path}") + resolved.pop(key, None) + resolved["prompt"] = prompt + else: + resolved[key] = str(path) + return resolved + + def _robot_control_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: try: payload = json.loads(dataset.path.read_text(encoding="utf-8")) @@ -145,9 +319,7 @@ def _robot_action_parity( raise QualificationError("robot-action min_sample_pass_rate must be in [0, 1]") thresholds = {key: value for key, value in gate.items() if key != "min_sample_pass_rate"} rows = [] - for sample, candidate_sample, reference_sample in zip( - selected, actual, expected, strict=True - ): + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): try: metrics = robot_action_metrics(candidate_sample, reference_sample) passed = robot_action_passes(metrics, thresholds) @@ -205,9 +377,7 @@ def _metric_geometry_parity( raise QualificationError("metric-geometry min_sample_pass_rate must be in [0, 1]") thresholds = {key: value for key, value in gate.items() if key != "min_sample_pass_rate"} rows = [] - for sample, candidate_sample, reference_sample in zip( - selected, actual, expected, strict=True - ): + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): try: metrics = metric_geometry_metrics(candidate_sample, reference_sample) passed = metric_geometry_passes(metrics, thresholds) @@ -264,9 +434,7 @@ def _stereo_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]] raise QualificationError(f"stereo request {index} has no {source}") image = (root / relative).resolve() if not image.is_relative_to(root) or not image.is_file(): - raise QualificationError( - f"stereo request {index} {source} is unavailable: {image}" - ) + raise QualificationError(f"stereo request {index} {source} is unavailable: {image}") sample[target] = str(image) selected.append(sample) return selected @@ -366,9 +534,7 @@ def _stereo_disparity_parity( } for sample in selected ] - actual, bundle = _candidate_outputs( - case, context, output, "disparity", candidate_requests - ) + actual, bundle = _candidate_outputs(case, context, output, "disparity", candidate_requests) gate = configured.get("gate", {}) if not isinstance(gate, Mapping): raise QualificationError("stereo Accuracy gate must be an object") @@ -386,9 +552,7 @@ def _stereo_disparity_parity( raise QualificationError("stereo error gates are invalid") rows = [] - for sample, candidate_sample, reference_sample in zip( - selected, actual, expected, strict=True - ): + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): candidate_values = _disparity_values(candidate_sample, "candidate") reference_values = _disparity_values(reference_sample, "reference") if len(candidate_values) != len(reference_values): @@ -396,15 +560,12 @@ def _stereo_disparity_parity( candidate_valid = all(math.isfinite(value) and value >= 0.0 for value in candidate_values) reference_valid = all(math.isfinite(value) and value >= 0.0 for value in reference_values) dot = math.fsum( - left * right - for left, right in zip(candidate_values, reference_values, strict=True) + left * right for left, right in zip(candidate_values, reference_values, strict=True) ) candidate_norm = math.sqrt(math.fsum(value * value for value in candidate_values)) reference_norm = math.sqrt(math.fsum(value * value for value in reference_values)) cosine = ( - dot / (candidate_norm * reference_norm) - if candidate_norm and reference_norm - else 0.0 + dot / (candidate_norm * reference_norm) if candidate_norm and reference_norm else 0.0 ) differences = [ abs(left - right) @@ -1288,7 +1449,29 @@ def _candidate_outputs( summary = cell.get("output_summary") if not isinstance(summary, Mapping): raise QualificationError("trtmc-bench omitted an Accuracy output") - outputs.append(dict(summary)) + resolved_summary = dict(summary) + artifact_dir = cell.get("artifact_dir") + if isinstance(artifact_dir, str) and artifact_dir: + artifact_root = (candidate_output / artifact_dir).resolve() + if not artifact_root.is_relative_to(candidate_output.resolve()): + raise QualificationError("trtmc-bench returned an unsafe artifact directory") + for name, value in tuple(resolved_summary.items()): + if name.endswith("_artifact") and isinstance(value, str): + path = Path(value) + resolved_summary[name] = str( + path if path.is_absolute() else (artifact_root / path).resolve() + ) + elif name.endswith("_artifacts") and isinstance(value, list): + resolved_summary[name] = [ + str( + Path(item) + if Path(item).is_absolute() + else (artifact_root / item).resolve() + ) + for item in value + if isinstance(item, str) + ] + outputs.append(resolved_summary) return outputs, bundle @@ -1579,17 +1762,19 @@ def _binary_masks(value: Mapping[str, Any], label: str) -> tuple[int, int, list[ try: numeric = [float(value) for value in masks] except (TypeError, ValueError) as error: - raise QualificationError( - f"{label} prompted-segmentation masks are not numeric" - ) from error + raise QualificationError(f"{label} prompted-segmentation masks are not numeric") from error if not all(math.isfinite(value) for value in numeric): raise QualificationError(f"{label} prompted-segmentation masks are not finite") threshold = 0.0 if kind == "logits" else 0.5 area = height * width - return height, width, [ - [value > threshold for value in numeric[index : index + area]] - for index in range(0, len(numeric), area) - ] + return ( + height, + width, + [ + [value > threshold for value in numeric[index : index + area]] + for index in range(0, len(numeric), area) + ], + ) def _mask_iou(left: Sequence[bool], right: Sequence[bool]) -> float: @@ -1598,7 +1783,9 @@ def _mask_iou(left: Sequence[bool], right: Sequence[bool]) -> float: return intersection / union if union else 1.0 -def _match_masks(candidate: Sequence[Sequence[bool]], reference: Sequence[Sequence[bool]]) -> list[float]: +def _match_masks( + candidate: Sequence[Sequence[bool]], reference: Sequence[Sequence[bool]] +) -> list[float]: matched_reference: set[int] = set() matches = [] for mask in candidate: @@ -1810,12 +1997,8 @@ def _text_prompted_instance_segmentation_parity( matches = _match_instances(candidate, reference) denominator = max(len(candidate[2]), len(reference[2]), 1) match_rate = len(matches) / denominator - minimum_sample_mask_iou = min( - (match["mask_iou"] for match in matches), default=0.0 - ) - minimum_sample_box_iou = min( - (match["box_iou"] for match in matches), default=0.0 - ) + minimum_sample_mask_iou = min((match["mask_iou"] for match in matches), default=0.0) + minimum_sample_box_iou = min((match["box_iou"] for match in matches), default=0.0) maximum_sample_score_error = max( (match["score_abs_error"] for match in matches), default=1.0 ) @@ -2016,10 +2199,7 @@ def _ocr_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: def _ocr_gold_match(text: str, answers: Sequence[str]) -> bool: normalized = _normalized_answer(text).strip(".,!?;:'\"") - return any( - normalized == _normalized_answer(answer).strip(".,!?;:'\"") - for answer in answers - ) + return any(normalized == _normalized_answer(answer).strip(".,!?;:'\"") for answer in answers) def _ocr_text_parity( @@ -2180,10 +2360,14 @@ def _localization_text_parity( maximum_point_distance = float(gate.get("max_localization_point_distance", 10.0)) maximum_text_distance = float(gate.get("max_normalized_edit_distance", 0.5)) minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) - if not all( - 0.0 <= value <= 1.0 - for value in (minimum_box_iou, maximum_text_distance, minimum_sample_rate) - ) or maximum_point_distance < 0.0 or not math.isfinite(maximum_point_distance): + if ( + not all( + 0.0 <= value <= 1.0 + for value in (minimum_box_iou, maximum_text_distance, minimum_sample_rate) + ) + or maximum_point_distance < 0.0 + or not math.isfinite(maximum_point_distance) + ): raise QualificationError("localization gates are outside their valid ranges") rows = [] @@ -2206,7 +2390,9 @@ def _localization_text_parity( localization_passed = ( alignment >= minimum_box_iou if candidate_kind == "box" - else alignment <= maximum_point_distance if candidate_kind == "point" else False + else alignment <= maximum_point_distance + if candidate_kind == "point" + else False ) rows.append( { diff --git a/tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml b/tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml new file mode 100644 index 0000000000..7ba248d3a7 --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml b/tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml new file mode 100644 index 0000000000..4ab9b70fec --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/media_generation_performance.yaml b/tools/benchmark_qualification/benchmarks/media_generation_performance.yaml new file mode 100644 index 0000000000..4ab9b70fec --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/media_generation_performance.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/task_output_parity.yaml b/tools/benchmark_qualification/benchmarks/task_output_parity.yaml new file mode 100644 index 0000000000..61affffefb --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/task_output_parity.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Family profiles own the request and official reference. Generated tasks do +# not require an unrelated external dataset: the prompt/assets are the case. +kind: accuracy +input: + adapter: configured_request +metric: + name: task_output_parity diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 25e0355a78..90ce84ee5c 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -5,7 +5,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field as dataclass_field from pathlib import Path import re from typing import Any, Mapping, Sequence @@ -30,6 +30,7 @@ class QualificationCase: values: Mapping[str, Any] source: Path reference_requirements: Path | None + reference_paths: Mapping[str, str] = dataclass_field(default_factory=dict) reference_build_isolation: bool = True environment_hook: Path | None = None @@ -55,6 +56,7 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: candidate = _candidate(raw.get("candidate"), family, path) reference_environment = raw.get("reference_environment") requirements = _requirements(reference_environment, path) + reference_paths = _reference_paths(reference_environment, path) build_isolation = _build_isolation(reference_environment, path) hook_path = path.parent / "prepare_environment.py" environment_hook = hook_path.resolve() if hook_path.is_file() else None @@ -77,6 +79,7 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: values=dict(value), source=path.resolve(), reference_requirements=requirements, + reference_paths=reference_paths, reference_build_isolation=build_isolation, environment_hook=environment_hook, ) @@ -184,12 +187,31 @@ def _build_isolation(raw: Any, path: Path) -> bool: raise QualificationError(f"{path}: reference_environment must be an object") value = raw.get("build_isolation", True) if not isinstance(value, bool): - raise QualificationError( - f"{path}: reference_environment.build_isolation must be a boolean" - ) + raise QualificationError(f"{path}: reference_environment.build_isolation must be a boolean") return value +def _reference_paths(raw: Any, path: Path) -> dict[str, str]: + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise QualificationError(f"{path}: reference_environment must be an object") + configured = raw.get("paths", {}) + if not isinstance(configured, Mapping): + raise QualificationError(f"{path}: reference_environment.paths must be an object") + result = {} + for name, value in configured.items(): + if not isinstance(name, str) or not name: + raise QualificationError(f"{path}: reference_environment.paths keys must be strings") + relative = Path(_string(value, f"reference_environment.paths.{name}", path)) + if relative.is_absolute() or ".." in relative.parts: + raise QualificationError( + f"{path}: reference_environment.paths.{name} must stay inside the environment" + ) + result[name] = relative.as_posix() + return result + + def _yaml_object(path: Path, label: str) -> dict[str, Any]: try: value = yaml.safe_load(path.read_text(encoding="utf-8")) diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py index 3f1faa1d4c..bd36ae680f 100644 --- a/tools/benchmark_qualification/performance.py +++ b/tools/benchmark_qualification/performance.py @@ -17,6 +17,7 @@ from .runtime import ( RuntimeContext, benchmark_executable, + reference_environment_paths, reference_python, require_candidate, run_command, @@ -67,8 +68,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st measurement = configured.get("measurement") reference_timing = definition.get("reference_timing") if not all( - isinstance(value, Mapping) - for value in (request, baseline, measurement, reference_timing) + isinstance(value, Mapping) for value in (request, baseline, measurement, reference_timing) ): raise QualificationError( "Performance request, reference, measurement, and reference timing must be objects" @@ -77,7 +77,18 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st assert isinstance(baseline, Mapping) assert isinstance(measurement, Mapping) assert isinstance(reference_timing, Mapping) + stability = definition.get("stability", {}) + required_samples = stability.get("samples") if isinstance(stability, Mapping) else None + if ( + isinstance(required_samples, bool) + or not isinstance(required_samples, int) + or int(measurement.get("iterations", 10)) < required_samples + ): + raise QualificationError( + "Performance measurement.iterations must satisfy the stability sample count" + ) request = _resolve_family_assets(case, request) + baseline = _resolve_reference_assets(case, baseline) descriptor = write_model_descriptor(case, output, request, context=context) entry_id = f"qualification.{case.family}.{case.name}" suite = { @@ -105,7 +116,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st } ], } - if definition.get("stability") != { + if stability != { "samples": 10, "max_half_median_change_percent": 5.0, "median_band_percent": 5.0, @@ -125,6 +136,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st "personaplex_repo": "", "fast_foundation_stereo_model": "", } + references.update(reference_environment_paths(case, context)) environment = { "schema_version": "trtmc.perf-environment/v2", "name": "qualification", @@ -222,9 +234,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st return result -def _resolve_family_assets( - case: QualificationCase, request: Mapping[str, Any] -) -> dict[str, Any]: +def _resolve_family_assets(case: QualificationCase, request: Mapping[str, Any]) -> dict[str, Any]: resolved = dict(request) family_root = case.source.parents[2].resolve() for key, value in request.items(): @@ -238,5 +248,41 @@ def _resolve_family_assets( raise QualificationError( f"Performance request asset {key!r} is unavailable inside {family_root}: {path}" ) - resolved[key] = str(path) + if key == "prompt_path": + payload = ( + json.loads(path.read_text(encoding="utf-8")) if path.suffix == ".json" else None + ) + prompt = ( + payload.get("prompt") + if isinstance(payload, Mapping) + else path.read_text(encoding="utf-8").strip() + ) + if not isinstance(prompt, str) or not prompt: + raise QualificationError(f"Performance prompt is unavailable: {path}") + resolved.pop(key, None) + resolved["prompt"] = prompt + else: + resolved[key] = str(path) + return resolved + + +def _resolve_reference_assets( + case: QualificationCase, reference: Mapping[str, Any] +) -> dict[str, Any]: + resolved = dict(reference) + options = reference.get("adapter_options", {}) + if not isinstance(options, Mapping): + raise QualificationError("Performance reference.adapter_options must be an object") + resolved_options = dict(options) + family_root = case.source.parents[2].resolve() + for key, value in options.items(): + if not key.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if not path.is_absolute(): + path = (case.source.parent / path).resolve() + if not path.is_relative_to(family_root) or not path.is_file(): + raise QualificationError(f"Performance reference asset {key!r} is unavailable: {path}") + resolved_options[key] = str(path) + resolved["adapter_options"] = resolved_options return resolved diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 92f1f5c68b..9d1e99ea35 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -264,14 +264,27 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: return python +def reference_environment_paths(case: QualificationCase, context: RuntimeContext) -> dict[str, str]: + """Resolve family-declared reference inputs below its isolated environment.""" + if not case.reference_paths: + return {} + environment = reference_python(case, context).parent.parent.resolve() + result = {} + for name, relative in case.reference_paths.items(): + path = (environment / relative).resolve() + if not path.is_relative_to(environment) or not path.exists(): + raise QualificationError(f"prepared reference path {name!r} is unavailable: {path}") + result[name] = str(path) + return result + + def benchmark_executable(case: QualificationCase, context: RuntimeContext) -> Path: """Run user-facing benchmark and bundle build with the family Python environment.""" python = reference_python(case, context) digest = hashlib.sha256(f"{python.resolve()}\0{context.trtmc_bench}".encode()).hexdigest()[:12] launcher = context.environment_root / "launchers" / f"{case.family}-{digest}" contents = ( - "#!/bin/sh\n" - f"exec {shlex.quote(str(python))} {shlex.quote(str(context.trtmc_bench))} \"$@\"\n" + f'#!/bin/sh\nexec {shlex.quote(str(python))} {shlex.quote(str(context.trtmc_bench))} "$@"\n' ) if not launcher.is_file() or launcher.read_text(encoding="utf-8") != contents: launcher.parent.mkdir(parents=True, exist_ok=True) @@ -289,11 +302,7 @@ def _inherit_parent_site_packages(environment: Path) -> None: f"reference environment has no unambiguous site-packages directory: {environment}" ) parent_packages = sorted( - { - str(Path(value).resolve()) - for value in site.getsitepackages() - if Path(value).is_dir() - } + {str(Path(value).resolve()) for value in site.getsitepackages() if Path(value).is_dir()} ) if not parent_packages or any("\n" in value for value in parent_packages): raise QualificationError("cannot resolve parent Python site-packages") diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 0a47a08227..2f071699af 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -1844,8 +1844,12 @@ def test_shared_definitions_own_dataset_and_metric_not_models() -> None: assert "models" not in definition assert definition["kind"] == case.kind if case.kind == "accuracy": - assert definition["dataset"]["id"] assert definition["metric"]["name"] + if definition["metric"]["name"] == "task_output_parity": + assert definition["input"]["adapter"] == "configured_request" + assert "dataset" not in definition + else: + assert definition["dataset"]["id"] def test_shared_performance_definitions_own_complete_reference_timing() -> None: From 2a4b5bf978c8540c1150c8cfdfa520eab797eae4 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 14:51:38 +0000 Subject: [PATCH 070/100] fix(benchmark): preserve Bark prompts Signed-off-by: chaofengw --- families/bark/tests/benchmark/bark-large.yaml | 2 +- families/bark/tests/benchmark/bark-small.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/families/bark/tests/benchmark/bark-large.yaml b/families/bark/tests/benchmark/bark-large.yaml index fe692f9e20..f48edd2a50 100644 --- a/families/bark/tests/benchmark/bark-large.yaml +++ b/families/bark/tests/benchmark/bark-large.yaml @@ -8,7 +8,7 @@ accuracy: - name: generated-audio-parity benchmark: task_output_parity operation: generate_audio - request: &request {prompt: Hello, this is a test of the text to speech system., max_new_tokens: 768, seed: 42} + request: &request {prompt: "Hello, this is a test of the text to speech system.", max_new_tokens: 768, seed: 42} reference: &reference {adapter: hf-transformers-tts, mode: hf-eager, precision: fp32, output_contract: audio-artifact-parity} gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} performance: diff --git a/families/bark/tests/benchmark/bark-small.yaml b/families/bark/tests/benchmark/bark-small.yaml index 54a7e23e15..9bee42731d 100644 --- a/families/bark/tests/benchmark/bark-small.yaml +++ b/families/bark/tests/benchmark/bark-small.yaml @@ -8,7 +8,7 @@ accuracy: - name: generated-audio-parity benchmark: task_output_parity operation: generate_audio - request: &request {prompt: Hello, this is a test of the audio generation system., max_new_tokens: 768, seed: 0} + request: &request {prompt: "Hello, this is a test of the audio generation system.", max_new_tokens: 768, seed: 0} reference: &reference {adapter: hf-transformers-tts, mode: hf-eager, precision: fp16, output_contract: audio-artifact-parity} gate: &gate {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} performance: From ec6947d0f1851754d58ed1c4f303021139d9a98c Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 15:04:49 +0000 Subject: [PATCH 071/100] fix(ci): align benchmark contract tests Signed-off-by: chaofengw --- .../tests/test_diffusers_semantic_reference.py | 7 +++++-- tools/tests/test_architecture.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py b/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py index 8655d97073..49aab38b1a 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py @@ -172,8 +172,11 @@ def test_prompt_and_seed_order_reaches_every_pipeline_invocation(recording_diffu assert all(call["observed_seeds"] == expected_seeds for call in recorded.calls) assert [generator.resets for generator in recorded.generators] == [[seed] * 4 for seed in expected_seeds] assert all(generator.device == "cuda" for generator in recorded.generators) - assert all(summary == {"media_type": "image", "media_count": len(expected_seeds), - "height": 8, "width": 10, "channels": 3, "finite": True} for summary in summaries) + for summary in summaries: + media = summary.pop("_media") + assert np.asarray(media).shape == (len(expected_seeds), 8, 10, 3) + assert summary == {"media_type": "image", "media_count": len(expected_seeds), + "height": 8, "width": 10, "channels": 3, "finite": True} assert payload == source diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 3a50bd948b..16aef8e646 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -513,6 +513,8 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/test_impact.py", "tools/benchmark_qualification/__init__.py", "tools/benchmark_qualification/accuracy.py", + "tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml", + "tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml", "tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml", "tools/benchmark_qualification/benchmarks/embedding_performance.yaml", "tools/benchmark_qualification/benchmarks/encoder_performance.yaml", @@ -530,6 +532,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", + "tools/benchmark_qualification/benchmarks/media_generation_performance.yaml", "tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml", "tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml", "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", @@ -541,6 +544,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", "tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", + "tools/benchmark_qualification/benchmarks/task_output_parity.yaml", "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", "tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml", From 1f3d134d6ec41d9df1e7f1535a00756a79761563 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 15:23:02 +0000 Subject: [PATCH 072/100] fix(benchmark): preserve scalar image requests Dispatch by the request prompt shape so a task that implements both scalar and batch image interfaces does not reinterpret a scalar prompt as an array. Cover the dual-interface case in the native worker E2E fixture. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 12 +++++----- .../native/test_benchmark_worker_e2e.cpp | 21 ++++++++++++++++++ core/runtime/tests/fake_family.cpp | 22 +++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 17b72052c5..55e9c27b02 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -278,18 +278,20 @@ Json image_observation(const std::vector& results) { Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& timing) { const auto config = image_config(request); - const std::string prompt = - request.at("prompt").is_array() ? "" : request.at("prompt").get(); + const bool batch_request = request.at("prompt").is_array(); + const std::string prompt = batch_request ? "" : request.at("prompt").get(); std::function()> invoke; std::optional cached; - if (auto* batch = dynamic_cast(&task)) { + if (batch_request) { + auto& batch = + require_interface(task, "IImageBatchGeneration"); const auto prompts = request.at("prompt").get>(); auto seeds = optional_value>(request, "seeds", {}); if (seeds.empty()) seeds.assign(prompts.size(), static_cast(std::max(config.seed, 0))); - invoke = [batch, prompts, seeds, config]() { - return batch->generate_image_batch(prompts, seeds, config); + invoke = [&batch, prompts, seeds, config]() { + return batch.generate_image_batch(prompts, seeds, config); }; } else if (auto* edit = dynamic_cast(&task)) { const std::string path = request.at("image_path").get(); diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index a291a6ba88..df112111ae 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -159,6 +159,27 @@ int main(int argc, char** argv) { std::filesystem::remove(vector_bundle); } + const auto image_bundle = runtime_root / "benchmark_fake_image_generation.bundle"; + write_bundle(image_bundle, "image_generation"); + Json image_request = request; + image_request["case_name"] = "fake-image-generation"; + image_request["bundle"] = image_bundle.string(); + image_request["operation"] = "generate_image"; + image_request["request"] = {{"prompt", "Hello"}}; + { + std::ofstream request_file(request_path); + request_file << image_request << '\n'; + } + check(std::system(command.c_str()) == 0, + "single prompt uses the scalar interface when a task also supports batching"); + std::ifstream image_output_file(output_path); + Json image_result; + image_output_file >> image_result; + check(image_result.at("status") == "completed" && + image_result.at("output_summary").at("generated_images") == 1, + "single image generation returns one result"); + std::filesystem::remove(image_bundle); + const auto legacy_image = runtime_root / "benchmark_legacy_segment.ppm"; { std::ofstream image(legacy_image, std::ios::binary); diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index f923cd40e5..9327a8e643 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -56,6 +56,26 @@ class FakeEmbedding final : public trtmc::IEmbedding { } }; +class FakeImageGeneration final : public trtmc::IImageGeneration, + public trtmc::IImageBatchGeneration { + public: + const char* task() const noexcept override { return trtmc::IImageGeneration::kTask; } + + trtmc::ImageResult generate_image(const std::string&, + const trtmc::ImageGenerationConfig&) override { + return {{0.25F, 0.5F, 0.75F}, 1, 1, 3, 1}; + } + + std::vector + generate_image_batch(const std::vector& prompts, const std::vector&, + const trtmc::ImageGenerationConfig& config) override { + std::vector results; + for (const auto& prompt : prompts) + results.push_back(generate_image(prompt, config)); + return results; + } +}; + class FakeSegmentation final : public trtmc::ISegmentation { public: trtmc::SegmentResult segment(const float*, std::int32_t height, std::int32_t width) override { @@ -126,6 +146,8 @@ trtmc::ITask* create_fake_task(const trtmc::FamilyContext& context) { return new FakeEncoding(); if (context.reader.info().task == trtmc::IEmbedding::kTask) return new FakeEmbedding(); + if (context.reader.info().task == trtmc::IImageGeneration::kTask) + return new FakeImageGeneration(); if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) return new FakeForecast(context.backend, context.kv_cache_size_bytes); if (context.reader.info().task == trtmc::ITextGeneration::kTask) { From 4fb90e3309c625673b058d034db2a3d026445441 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 15:41:38 +0000 Subject: [PATCH 073/100] fix(benchmark): retain legacy media outputs Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 62 +++++++++++++++++-- .../native/test_benchmark_worker_e2e.cpp | 26 ++++++++ core/runtime/tests/fake_family.cpp | 10 +++ 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 55e9c27b02..e3fbc62841 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -265,6 +265,7 @@ Json image_observation(const std::vector& results) { Json value = {{"generated_images", results.size()}, {"batch_size", results.size()}, {"generated_frames", frames}, + {"media_count", frames}, {"output_elements", pixels}}; if (!results.empty()) { value["height"] = results.front().height; @@ -276,6 +277,35 @@ Json image_observation(const std::vector& results) { return value; } +auto legacy_image_artifact_writer(std::string prefix) { + return [prefix = std::move(prefix), + iteration = std::uint64_t{0}](const std::vector& results) mutable { + ++iteration; + Json paths = Json::array(); + std::size_t artifact_index = 0; + for (const auto& result : results) { + if (result.height <= 0 || result.width <= 0 || result.channels <= 0 || + result.num_frames <= 0) + throw std::runtime_error("generated media has invalid dimensions"); + const auto frame_elements = + static_cast(result.height) * result.width * result.channels; + const auto expected = frame_elements * static_cast(result.num_frames); + if (expected != result.pixels.size()) + throw std::runtime_error("generated media has inconsistent pixel storage"); + for (int frame = 0; frame < result.num_frames; ++frame) { + const auto path = prefix + "." + std::to_string(iteration) + "." + + std::to_string(artifact_index++) + ".png"; + const auto offset = static_cast(frame_elements) * frame; + trtmc::cli::io::save_png( + path, {result.pixels.data() + offset, static_cast(frame_elements)}, + result.width, result.height, result.channels); + paths.push_back(std::filesystem::path(path).filename().string()); + } + } + return paths; + }; +} + Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& timing) { const auto config = image_config(request); const bool batch_request = request.at("prompt").is_array(); @@ -333,7 +363,15 @@ Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& t return std::vector{image.generate_image(prompt, config)}; }; } - return measure(timing, invoke, image_observation); + auto write_images = + legacy_image_artifact_writer(request.at("_artifact_prefix").get()); + return measure(timing, invoke, [&](const auto& results) { + auto output = image_observation(results); + output[output.value("media_type", "image") == "video" ? "frame_artifacts" + : "image_artifacts"] = + write_images(results); + return output; + }); } Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& timing) { @@ -344,9 +382,15 @@ Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& t optional_value(request, "talker_max_new_tokens", 0); config.seed = optional_value(request, "seed", -1); const std::string prompt = request.at("prompt").get(); + auto write_audio = [prefix = request.at("_artifact_prefix").get(), + iteration = std::uint64_t{0}](const trtmc::AudioResult& result) mutable { + const auto path = prefix + "." + std::to_string(++iteration) + ".wav"; + trtmc::cli::io::write_wav(result, path); + return std::filesystem::path(path).filename().string(); + }; return measure( timing, [&]() { return interface.generate_audio(prompt, config); }, - [](const trtmc::AudioResult& result) { + [&](const trtmc::AudioResult& result) { const double seconds = result.sample_rate > 0 ? static_cast(result.samples.size()) / result.sample_rate @@ -354,7 +398,8 @@ Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& t return Json{{"output_samples", result.samples.size()}, {"num_samples", result.samples.size()}, {"output_audio_seconds", seconds}, - {"sample_rate", result.sample_rate}}; + {"sample_rate", result.sample_rate}, + {"audio_artifact", write_audio(result)}}; }); } @@ -368,6 +413,12 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) { config.max_new_tokens = optional_value(request, "max_new_tokens", 50); config.seed = optional_value(request, "seed", -1); config.tail_frames = optional_value(request, "tail_frames", 0); + auto write_audio = [prefix = request.at("_artifact_prefix").get(), + iteration = std::uint64_t{0}](const trtmc::AudioResult& result) mutable { + const auto path = prefix + "." + std::to_string(++iteration) + ".wav"; + trtmc::cli::io::write_wav(result, path); + return std::filesystem::path(path).filename().string(); + }; return measure( timing, [&]() { @@ -381,7 +432,7 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) { audio.sample_rate), static_cast(audio.samples.size()) / audio.sample_rate}; }, - [](const auto& value) { + [&](const auto& value) { const auto& result = value.first; return Json{{"input_audio_seconds", value.second}, {"output_audio_seconds", @@ -390,7 +441,8 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) { : 0.0}, {"output_samples", result.samples.size()}, {"num_samples", result.samples.size()}, - {"sample_rate", result.sample_rate}}; + {"sample_rate", result.sample_rate}, + {"audio_artifact", write_audio(result)}}; }); } diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index df112111ae..9edd00cf92 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -178,8 +178,34 @@ int main(int argc, char** argv) { check(image_result.at("status") == "completed" && image_result.at("output_summary").at("generated_images") == 1, "single image generation returns one result"); + const auto image_artifacts = image_result.at("output_summary").at("image_artifacts"); + check(image_artifacts.size() == 1 && + std::filesystem::is_regular_file(runtime_root / image_artifacts.at(0)), + "legacy image generation retains the actual PNG outside timed execution"); std::filesystem::remove(image_bundle); + const auto audio_bundle = runtime_root / "benchmark_fake_audio_generation.bundle"; + write_bundle(audio_bundle, "audio_generation"); + Json audio_request = request; + audio_request["case_name"] = "fake-audio-generation"; + audio_request["bundle"] = audio_bundle.string(); + audio_request["operation"] = "generate_audio"; + audio_request["request"] = {{"prompt", "Hello"}}; + { + std::ofstream request_file(request_path); + request_file << audio_request << '\n'; + } + check(std::system(command.c_str()) == 0, "legacy audio worker process completed"); + std::ifstream audio_output_file(output_path); + Json audio_result; + audio_output_file >> audio_result; + const auto audio_artifact = + audio_result.at("output_summary").at("audio_artifact").get(); + check(audio_result.at("status") == "completed" && + std::filesystem::is_regular_file(runtime_root / audio_artifact), + "legacy audio generation retains the actual WAV outside timed execution"); + std::filesystem::remove(audio_bundle); + const auto legacy_image = runtime_root / "benchmark_legacy_segment.ppm"; { std::ofstream image(legacy_image, std::ios::binary); diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 9327a8e643..721099a849 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -76,6 +76,14 @@ class FakeImageGeneration final : public trtmc::IImageGeneration, } }; +class FakeAudioGeneration final : public trtmc::IAudioGeneration { + public: + trtmc::AudioResult generate_audio(const std::string&, + const trtmc::AudioGenerationConfig&) override { + return {{0.25F, -0.5F, 0.75F, -1.0F}, 4, 8000}; + } +}; + class FakeSegmentation final : public trtmc::ISegmentation { public: trtmc::SegmentResult segment(const float*, std::int32_t height, std::int32_t width) override { @@ -148,6 +156,8 @@ trtmc::ITask* create_fake_task(const trtmc::FamilyContext& context) { return new FakeEmbedding(); if (context.reader.info().task == trtmc::IImageGeneration::kTask) return new FakeImageGeneration(); + if (context.reader.info().task == trtmc::IAudioGeneration::kTask) + return new FakeAudioGeneration(); if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) return new FakeForecast(context.backend, context.kv_cache_size_bytes); if (context.reader.info().task == trtmc::ITextGeneration::kTask) { From c706dd3b96941eb4787cc4de2eae5ad6ba8f4a4b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 16:24:21 +0000 Subject: [PATCH 074/100] fix(benchmark): bound reference extension builds Signed-off-by: chaofengw --- families/lance/tests/benchmark/lance-3b-x2t-image.yaml | 1 + tools/benchmark_qualification/runtime.py | 6 ++++++ tools/tests/test_model_benchmark.py | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml index b4afc10818..b891f7c119 100644 --- a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml +++ b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml @@ -5,6 +5,7 @@ model: lance-3b-x2t-image candidate: {family: lance, checkpoint: bytedance-research/Lance, revision: 7395315758865e6f56ab87ad06a88c7ac172f056, task: vision_language_generation, precision: bf16, bundle: lance-3b-x2t-image.bundle, build: {max_sequence_length: 384}} reference_environment: requirements: ../../requirements.txt + build_isolation: false paths: {lance_repo: trtmc-reference/Lance} accuracy: - name: generated-text-parity diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 9d1e99ea35..63270aa505 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -241,12 +241,18 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: if not case.reference_build_isolation: install_command.append("--no-build-isolation") install_command.extend(("-r", str(requirements))) + install_environment = dict(os.environ) + # Source distributions commonly delegate extension builds to Ninja. Keep one + # family environment from exhausting a shared validation host while allowing + # operators to select a different limit explicitly. + install_environment.setdefault("MAX_JOBS", "4") installed = run_command( install_command, setup_root, "pip", timeout=1800, verbose=context.verbose, + env=install_environment, ) if installed.returncode != 0: raise QualificationError(f"cannot install reference requirements; see {setup_root}") diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 2f071699af..1dabb80546 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -51,6 +51,8 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert not any("l0" in case.model.lower() for case in cases) nemotron_h = [case for case in cases if case.model == "nemotron-h-nano-9b"] assert nemotron_h and all(not case.reference_build_isolation for case in nemotron_h) + lance = [case for case in cases if case.model == "lance-3b-x2t-image"] + assert lance and all(not case.reference_build_isolation for case in lance) def test_timm_qualification_profiles_use_the_family_build_task() -> None: @@ -2112,9 +2114,11 @@ def test_family_reference_environment_inherits_parent_venv_packages( parent_packages = tmp_path / "parent-venv/site-packages" parent_packages.mkdir(parents=True) commands = [] + environments = [] def complete(command, *_args, **_kwargs): commands.append(command) + environments.append(_kwargs.get("env")) if command[1:3] == ["-m", "venv"]: environment = Path(command[-1]) (environment / "bin").mkdir(parents=True) @@ -2137,6 +2141,8 @@ def complete(command, *_args, **_kwargs): assert inherited.read_text(encoding="utf-8") == f"{parent_packages.resolve()}\n" pip_command = next(command for command in commands if command[1:3] == ["-m", "pip"]) assert "--no-build-isolation" in pip_command + pip_index = commands.index(pip_command) + assert environments[pip_index]["MAX_JOBS"] == "4" def test_family_environment_hook_runs_after_requirements_install( From 14b0d231f592642568cff8114d6d56d91e89ad61 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 18:08:00 +0000 Subject: [PATCH 075/100] fix(benchmark): install PersonaPlex audio runtime Install the official sphn dependency in the family-owned reference environment so isolated Accuracy and Performance runners can load the PersonaPlex pipeline. Signed-off-by: chaofengw --- families/personaplex/requirements.txt | 1 + tools/tests/test_model_benchmark.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/families/personaplex/requirements.txt b/families/personaplex/requirements.txt index eb024b6d84..4e802a1e66 100644 --- a/families/personaplex/requirements.txt +++ b/families/personaplex/requirements.txt @@ -4,3 +4,4 @@ # Match the tensor rearrangement dependency in the pinned official Moshi source. einops==0.7.0 soundfile +sphn==0.1.4 diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 1dabb80546..ff7d83d5cf 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -55,6 +55,14 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert lance and all(not case.reference_build_isolation for case in lance) +def test_personaplex_reference_environment_installs_official_audio_dependency() -> None: + requirements = ( + REPOSITORY / "families/personaplex/requirements.txt" + ).read_text(encoding="utf-8") + + assert "sphn==0.1.4" in requirements.splitlines() + + def test_timm_qualification_profiles_use_the_family_build_task() -> None: cases = discover(REPOSITORY) profiles = { From 6e372821bd8820111f82235ac7f5fa5893316820 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 19:23:40 +0000 Subject: [PATCH 076/100] fix(benchmark): route text-only image requests Select legacy image editing and world-model interfaces only when the request supplies a conditioning image. This keeps multi-interface pipelines on text-to-image generation for text-only requests. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 19 ++++++++++++------- .../native/test_benchmark_worker_e2e.cpp | 2 +- core/runtime/tests/fake_family.cpp | 6 ++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index e3fbc62841..aab8a77fad 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -323,24 +323,29 @@ Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& t invoke = [&batch, prompts, seeds, config]() { return batch.generate_image_batch(prompts, seeds, config); }; - } else if (auto* edit = dynamic_cast(&task)) { + } else if (request.contains("image_path") && + dynamic_cast(&task) != nullptr) { + auto& edit = require_interface(task, "IImageEditing"); const std::string path = request.at("image_path").get(); if (!timing.asset_loading_included) cached = read_image(path); - invoke = [edit, prompt, path, config, &cached]() { + invoke = [&edit, prompt, path, config, &cached]() { if (cached) { - return std::vector{edit->generate_image( + return std::vector{edit.generate_image( prompt, cached->pixels.data(), cached->height, cached->width, config)}; } const Image image = read_image(path); - return std::vector{edit->generate_image( + return std::vector{edit.generate_image( prompt, image.pixels.data(), image.height, image.width, config)}; }; - } else if (auto* world = dynamic_cast(&task)) { + } else if (request.contains("image_path") && + dynamic_cast(&task) != nullptr) { + auto& world = + require_interface(task, "IWorldModelGeneration"); const std::string path = request.at("image_path").get(); if (!timing.asset_loading_included) cached = read_image(path); - invoke = [world, prompt, path, config, request, &cached]() { + invoke = [&world, prompt, path, config, request, &cached]() { std::optional loaded; if (!cached) loaded = read_image(path); @@ -355,7 +360,7 @@ Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& t optional_value>(request, "camera_intrinsics", {}); value.num_frames = optional_value(request, "num_frames", 0); value.generation = config; - return std::vector{world->generate_world(value)}; + return std::vector{world.generate_world(value)}; }; } else { auto& image = require_interface(task, "IImageGeneration"); diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index 9edd00cf92..d0f5debcec 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -171,7 +171,7 @@ int main(int argc, char** argv) { request_file << image_request << '\n'; } check(std::system(command.c_str()) == 0, - "single prompt uses the scalar interface when a task also supports batching"); + "text-only prompt uses generation when a task also supports editing and batching"); std::ifstream image_output_file(output_path); Json image_result; image_output_file >> image_result; diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 721099a849..db195081c2 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -57,6 +57,7 @@ class FakeEmbedding final : public trtmc::IEmbedding { }; class FakeImageGeneration final : public trtmc::IImageGeneration, + public trtmc::IImageEditing, public trtmc::IImageBatchGeneration { public: const char* task() const noexcept override { return trtmc::IImageGeneration::kTask; } @@ -66,6 +67,11 @@ class FakeImageGeneration final : public trtmc::IImageGeneration, return {{0.25F, 0.5F, 0.75F}, 1, 1, 3, 1}; } + trtmc::ImageResult generate_image(const std::string&, const float*, std::int32_t, std::int32_t, + const trtmc::ImageGenerationConfig&) override { + return {{0.75F, 0.5F, 0.25F}, 1, 1, 3, 1}; + } + std::vector generate_image_batch(const std::vector& prompts, const std::vector&, const trtmc::ImageGenerationConfig& config) override { From 78c57081b7d62d523ba4060a8a89e154b8063e72 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 20:06:01 +0000 Subject: [PATCH 077/100] fix(benchmark): localize Sana reference weights Materialize the pinned SANA-WM snapshot inside its family-owned Python environment using hard links when possible. This preserves reference path isolation without duplicating the shared cache on the same filesystem. Signed-off-by: chaofengw --- .../tests/benchmark/prepare_environment.py | 32 +++++++++++++++++-- .../benchmark/sana-wm-bidirectional.yaml | 2 +- .../sana_wm/tests/test_reference_contract.py | 25 +++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/families/sana_wm/tests/benchmark/prepare_environment.py b/families/sana_wm/tests/benchmark/prepare_environment.py index 46da0d6d6a..a158aa1f8f 100644 --- a/families/sana_wm/tests/benchmark/prepare_environment.py +++ b/families/sana_wm/tests/benchmark/prepare_environment.py @@ -4,6 +4,7 @@ """Prepare pinned SANA source and a local HF-compatible model directory.""" from pathlib import Path +import os import shutil import subprocess import sys @@ -14,6 +15,29 @@ REPOSITORY = "https://github.com/NVlabs/Sana.git" REVISION = "59629fdf790850797cb657bad014fce432bd713d" MODEL = "Efficient-Large-Model/SANA-WM_bidirectional" +MODEL_REVISION = "e96271d77398def8ebb9fc595e7c0056dc625ab7" + + +def _materialize_snapshot(snapshot: Path, destination: Path) -> None: + temporary = Path(tempfile.mkdtemp(prefix="SANA-model.", dir=destination.parent)) + try: + for source in snapshot.rglob("*"): + target = temporary / source.relative_to(snapshot) + if source.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + if not source.is_file(): + continue + target.parent.mkdir(parents=True, exist_ok=True) + resolved = source.resolve(strict=True) + try: + os.link(resolved, target) + except OSError: + shutil.copy2(resolved, target) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise def main() -> None: @@ -42,8 +66,12 @@ def main() -> None: shutil.rmtree(temporary, ignore_errors=True) raise model = parent / "SANA-model" - if not model.exists(): - model.symlink_to(Path(snapshot_download(MODEL)).resolve(), target_is_directory=True) + if model.is_dir() and not model.is_symlink(): + return + if model.exists() or model.is_symlink(): + raise RuntimeError(f"incomplete SANA model directory: {model}") + snapshot = Path(snapshot_download(MODEL, revision=MODEL_REVISION)).resolve() + _materialize_snapshot(snapshot, model) if __name__ == "__main__": diff --git a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml index 1fa221f256..63ef16e8dc 100644 --- a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml +++ b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: sana-wm-bidirectional -candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} +candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, revision: e96271d77398def8ebb9fc595e7c0056dc625ab7, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} reference_environment: requirements: ../../requirements.txt paths: {sana_repo: trtmc-reference/Sana, sana_model: trtmc-reference/SANA-model} diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index bc92d94003..ab2fabbf34 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -10,6 +10,7 @@ import pytest from . import test_e2e as e2e +from .benchmark import prepare_environment def test_official_source_dependencies_are_family_owned() -> None: @@ -29,6 +30,30 @@ def test_official_source_dependencies_are_family_owned() -> None: } <= requirements +def test_qualification_snapshot_is_materialized_inside_family_environment( + tmp_path: Path, +) -> None: + cache = tmp_path / "cache" + blob = cache / "blobs/weights" + blob.parent.mkdir(parents=True) + blob.write_bytes(b"weights") + snapshot = cache / "snapshots/revision" + (snapshot / "dit").mkdir(parents=True) + (snapshot / "dit/model.safetensors").symlink_to(blob) + environment = tmp_path / "environment" + environment.mkdir() + destination = environment / "SANA-model" + + prepare_environment._materialize_snapshot(snapshot, destination) + + materialized = destination / "dit/model.safetensors" + assert destination.is_dir() and not destination.is_symlink() + assert destination.resolve().is_relative_to(environment) + assert materialized.read_bytes() == b"weights" + assert materialized.samefile(blob) + assert prepare_environment.MODEL_REVISION == "e96271d77398def8ebb9fc595e7c0056dc625ab7" + + def test_manifest_owns_the_exact_camera_control_workload() -> None: _, manifest, case = e2e.CASES["sana-wm-bidirectional"] assert manifest["video_num_frames"] == 321 From 5c3132b4e586977a37a2feed784750a6cb9357ec Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 20:49:17 +0000 Subject: [PATCH 078/100] fix(benchmark): use stable Z-Image reference precision Signed-off-by: chaofengw --- families/z_image/tests/benchmark/z-image-turbo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/families/z_image/tests/benchmark/z-image-turbo.yaml b/families/z_image/tests/benchmark/z-image-turbo.yaml index 4e6664f62e..d2a4d7a217 100644 --- a/families/z_image/tests/benchmark/z-image-turbo.yaml +++ b/families/z_image/tests/benchmark/z-image-turbo.yaml @@ -9,7 +9,7 @@ accuracy: benchmark: task_output_parity operation: generate_image request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 9, guidance_scale: 0.0, height: 1024, width: 1024} - reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: fp16, output_contract: media-artifact-parity} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity} gate: {min_psnr: 5.0, min_ssim: 0.1} performance: - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} From c6c7b21476ca760a9885b5e0f0cd6468aa6e5ece Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 21:53:40 +0000 Subject: [PATCH 079/100] fix(benchmark): support native reference builds Signed-off-by: chaofengw --- families/personaplex/requirements.txt | 1 - .../tests/benchmark/prepare_environment.py | 29 +++++++++++++++++++ tools/benchmark_qualification/runtime.py | 4 ++- tools/tests/test_model_benchmark.py | 25 ++++++++++++---- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/families/personaplex/requirements.txt b/families/personaplex/requirements.txt index 4e802a1e66..eb024b6d84 100644 --- a/families/personaplex/requirements.txt +++ b/families/personaplex/requirements.txt @@ -4,4 +4,3 @@ # Match the tensor rearrangement dependency in the pinned official Moshi source. einops==0.7.0 soundfile -sphn==0.1.4 diff --git a/families/personaplex/tests/benchmark/prepare_environment.py b/families/personaplex/tests/benchmark/prepare_environment.py index 9e043ac699..9f7b9140d0 100644 --- a/families/personaplex/tests/benchmark/prepare_environment.py +++ b/families/personaplex/tests/benchmark/prepare_environment.py @@ -3,6 +3,8 @@ """Prepare the pinned family-owned PersonaPlex reference checkout.""" +from importlib.metadata import PackageNotFoundError, version as package_version +import os from pathlib import Path import shutil import subprocess @@ -11,9 +13,36 @@ REPOSITORY = "https://github.com/NVIDIA/personaplex.git" REVISION = "3428dfd95309a7f3c84fd93259ded0f810d1ff91" +SPHN_VERSION = "0.1.4" + + +def _install_sphn() -> None: + try: + installed = package_version("sphn") + except PackageNotFoundError: + installed = None + if installed == SPHN_VERSION: + return + environment = dict(os.environ) + # sphn vendors an older Opus CMake project. CMake 4 requires its legacy + # policy floor to be selected explicitly when building on aarch64. + environment.setdefault("CMAKE_POLICY_VERSION_MINIMUM", "3.5") + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + f"sphn=={SPHN_VERSION}", + ], + check=True, + env=environment, + ) def main() -> None: + _install_sphn() parent = Path(sys.prefix) / "trtmc-reference" destination = parent / "personaplex" if (destination / "moshi").is_dir(): diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 63270aa505..63b020f1e2 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -250,7 +250,9 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: install_command, setup_root, "pip", - timeout=1800, + # Native reference dependencies such as FlashAttention can take more + # than 30 minutes to build on aarch64 even with bounded parallelism. + timeout=7200, verbose=context.verbose, env=install_environment, ) diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index ff7d83d5cf..83f254e7fc 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -16,6 +16,7 @@ from apps.benchmark.performance.baselines import hf_transformers, task_reference from apps.benchmark.performance.baselines.timing_contracts import timing_contract +from families.personaplex.tests.benchmark import prepare_environment as personaplex_environment from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification import performance as qualification_performance from tools.benchmark_qualification.catalog import ( @@ -55,12 +56,23 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert lance and all(not case.reference_build_isolation for case in lance) -def test_personaplex_reference_environment_installs_official_audio_dependency() -> None: - requirements = ( - REPOSITORY / "families/personaplex/requirements.txt" - ).read_text(encoding="utf-8") +def test_personaplex_reference_environment_installs_official_audio_dependency( + monkeypatch, +) -> None: + calls = [] + monkeypatch.setattr(personaplex_environment, "package_version", lambda _name: "0.1.3") + monkeypatch.setattr( + personaplex_environment.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)), + ) + + personaplex_environment._install_sphn() - assert "sphn==0.1.4" in requirements.splitlines() + command, kwargs = calls[0] + assert command[-1] == "sphn==0.1.4" + assert kwargs["check"] is True + assert kwargs["env"]["CMAKE_POLICY_VERSION_MINIMUM"] == "3.5" def test_timm_qualification_profiles_use_the_family_build_task() -> None: @@ -2123,10 +2135,12 @@ def test_family_reference_environment_inherits_parent_venv_packages( parent_packages.mkdir(parents=True) commands = [] environments = [] + timeouts = [] def complete(command, *_args, **_kwargs): commands.append(command) environments.append(_kwargs.get("env")) + timeouts.append(_kwargs.get("timeout")) if command[1:3] == ["-m", "venv"]: environment = Path(command[-1]) (environment / "bin").mkdir(parents=True) @@ -2151,6 +2165,7 @@ def complete(command, *_args, **_kwargs): assert "--no-build-isolation" in pip_command pip_index = commands.index(pip_command) assert environments[pip_index]["MAX_JOBS"] == "4" + assert timeouts[pip_index] == 7200 def test_family_environment_hook_runs_after_requirements_install( From dee033907ba0073791524ceffa3301395b0353bf Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 22:03:01 +0000 Subject: [PATCH 080/100] fix(benchmark): use headless OpenCV for SANA Signed-off-by: chaofengw --- .../tests/benchmark/prepare_environment.py | 25 +++++++++++++++++++ .../sana_wm/tests/test_reference_contract.py | 16 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/families/sana_wm/tests/benchmark/prepare_environment.py b/families/sana_wm/tests/benchmark/prepare_environment.py index a158aa1f8f..f6c86eaf14 100644 --- a/families/sana_wm/tests/benchmark/prepare_environment.py +++ b/families/sana_wm/tests/benchmark/prepare_environment.py @@ -16,6 +16,30 @@ REVISION = "59629fdf790850797cb657bad014fce432bd713d" MODEL = "Efficient-Large-Model/SANA-WM_bidirectional" MODEL_REVISION = "e96271d77398def8ebb9fc595e7c0056dc625ab7" +OPENCV_HEADLESS_VERSION = "4.11.0.86" + + +def _install_headless_opencv() -> None: + # mmcv declares the GUI OpenCV distribution, whose Linux wheel requires + # libxcb. The reference only uses image-processing APIs, so replace it + # with the headless wheel inside this family environment. + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=False, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + f"opencv-python-headless=={OPENCV_HEADLESS_VERSION}", + ], + check=True, + ) def _materialize_snapshot(snapshot: Path, destination: Path) -> None: @@ -41,6 +65,7 @@ def _materialize_snapshot(snapshot: Path, destination: Path) -> None: def main() -> None: + _install_headless_opencv() parent = Path(sys.prefix) / "trtmc-reference" source = parent / "Sana" parent.mkdir(parents=True, exist_ok=True) diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index ab2fabbf34..7e2efd3244 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -54,6 +54,22 @@ def test_qualification_snapshot_is_materialized_inside_family_environment( assert prepare_environment.MODEL_REVISION == "e96271d77398def8ebb9fc595e7c0056dc625ab7" +def test_qualification_environment_replaces_gui_opencv(monkeypatch) -> None: + calls = [] + monkeypatch.setattr( + prepare_environment.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)), + ) + + prepare_environment._install_headless_opencv() + + assert calls[0][0][-2:] == ["--yes", "opencv-python"] + assert calls[0][1]["check"] is False + assert calls[1][0][-1] == "opencv-python-headless==4.11.0.86" + assert calls[1][1]["check"] is True + + def test_manifest_owns_the_exact_camera_control_workload() -> None: _, manifest, case = e2e.CASES["sana-wm-bidirectional"] assert manifest["video_num_frames"] == 321 From d6a7adc63393514032cc8addea6bbdf6504cf562 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 23:27:26 +0000 Subject: [PATCH 081/100] fix(benchmark): prepare Lance image reference Signed-off-by: chaofengw --- .../tests/benchmark/prepare_environment.py | 18 ++++++++++++++++++ .../lance/tests/test_official_reference.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/families/lance/tests/benchmark/prepare_environment.py b/families/lance/tests/benchmark/prepare_environment.py index a60e66c148..8bbac437dc 100644 --- a/families/lance/tests/benchmark/prepare_environment.py +++ b/families/lance/tests/benchmark/prepare_environment.py @@ -13,10 +13,27 @@ REVISION = "4baeee086648996f6ab12e673cbe461b0b149997" +def _prepare_image_only_checkout(repository: Path) -> None: + dataset = repository / "data/datasets_custom/validation_dataset.py" + text = dataset.read_text(encoding="utf-8") + for line in ("import decord\n", "from decord import VideoReader\n"): + if line in text: + if text.count(line) != 1: + raise RuntimeError(f"Lance upstream import contract changed: {line.strip()}") + text = text.replace(line, "") + annotation = "video: VideoReader" + if annotation in text: + if text.count(annotation) != 1: + raise RuntimeError("Lance upstream VideoReader annotation contract changed") + text = text.replace(annotation, "video") + dataset.write_text(text, encoding="utf-8") + + def main() -> None: parent = Path(sys.prefix) / "trtmc-reference" destination = parent / "Lance" if (destination / "inference_lance.py").is_file(): + _prepare_image_only_checkout(destination) return if destination.exists(): raise RuntimeError(f"incomplete Lance reference: {destination}") @@ -36,6 +53,7 @@ def main() -> None: check=True, ) temporary.rename(destination) + _prepare_image_only_checkout(destination) except Exception: shutil.rmtree(temporary, ignore_errors=True) raise diff --git a/families/lance/tests/test_official_reference.py b/families/lance/tests/test_official_reference.py index 2cb7ef01cb..05a564d2dc 100644 --- a/families/lance/tests/test_official_reference.py +++ b/families/lance/tests/test_official_reference.py @@ -15,6 +15,7 @@ _source_dir, run_official_generation, ) +from families.lance.tests.benchmark import prepare_environment def test_reference_source_is_required_and_fail_closed(monkeypatch, tmp_path) -> None: @@ -60,6 +61,21 @@ def test_image_source_removes_only_the_eager_video_import(tmp_path) -> None: assert (image_source / "modeling").is_symlink() +def test_qualification_checkout_removes_only_the_eager_video_import(tmp_path) -> None: + source = tmp_path / "source" + source.mkdir() + _fake_source(source) + + prepare_environment._prepare_image_only_checkout(source) + prepare_environment._prepare_image_only_checkout(source) + + patched = (source / "data/datasets_custom/validation_dataset.py").read_text() + assert "import decord" not in patched + assert "from decord" not in patched + assert "video: VideoReader" not in patched + assert "return VideoReader, decord, video" in patched + + def test_result_requires_one_nonempty_answer(tmp_path) -> None: result = tmp_path / "result.json" result.write_text(json.dumps([{"answer": " White "}]), encoding="utf-8") From 208134fd569f1f021601b639a8681a9e0bbe4aba Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 23:49:19 +0000 Subject: [PATCH 082/100] fix(benchmark): stabilize Qwen Image reference timing Signed-off-by: chaofengw --- families/qwen_image/tests/benchmark/qwen-image-2512.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/families/qwen_image/tests/benchmark/qwen-image-2512.yaml b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml index cba81cfecf..b286f40404 100644 --- a/families/qwen_image/tests/benchmark/qwen-image-2512.yaml +++ b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml @@ -7,4 +7,4 @@ reference_environment: {requirements: ../../requirements.txt} accuracy: - {name: generated-media-parity, benchmark: task_output_parity, operation: generate_image, request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024}, reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}}, gate: {min_psnr: 5.0, min_ssim: 0.1}} performance: - - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 10, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} From eeccdd6c8086a70822f4b8c51219b9417ac54fa0 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sat, 19 Sep 2026 23:55:31 +0000 Subject: [PATCH 083/100] fix(benchmark): materialize SANA text encoder Signed-off-by: chaofengw --- .../tests/benchmark/prepare_environment.py | 24 +++++++++++++++++ .../manifests/sana-wm-bidirectional.json | 3 ++- families/sana_wm/tests/test_e2e.py | 8 +++++- .../sana_wm/tests/test_reference_contract.py | 27 +++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/families/sana_wm/tests/benchmark/prepare_environment.py b/families/sana_wm/tests/benchmark/prepare_environment.py index f6c86eaf14..63e055b64c 100644 --- a/families/sana_wm/tests/benchmark/prepare_environment.py +++ b/families/sana_wm/tests/benchmark/prepare_environment.py @@ -16,6 +16,8 @@ REVISION = "59629fdf790850797cb657bad014fce432bd713d" MODEL = "Efficient-Large-Model/SANA-WM_bidirectional" MODEL_REVISION = "e96271d77398def8ebb9fc595e7c0056dc625ab7" +STAGE1_TEXT_ENCODER = "Efficient-Large-Model/gemma-2-2b-it" +STAGE1_TEXT_ENCODER_REVISION = "569d9809d0c8b6722d4d31b5a77a2ec7a400650a" OPENCV_HEADLESS_VERSION = "4.11.0.86" @@ -64,6 +66,26 @@ def _materialize_snapshot(snapshot: Path, destination: Path) -> None: raise +def _materialize_stage1_text_encoder(model: Path) -> None: + destination = model / "stage1_text_encoder" + required = ( + destination / "config.json", + destination / "model.safetensors.index.json", + destination / "tokenizer.json", + ) + if all(path.is_file() for path in required): + return + if destination.exists() or destination.is_symlink(): + raise RuntimeError(f"incomplete SANA stage-1 text encoder: {destination}") + snapshot = Path( + snapshot_download( + STAGE1_TEXT_ENCODER, + revision=STAGE1_TEXT_ENCODER_REVISION, + ) + ).resolve() + _materialize_snapshot(snapshot, destination) + + def main() -> None: _install_headless_opencv() parent = Path(sys.prefix) / "trtmc-reference" @@ -92,11 +114,13 @@ def main() -> None: raise model = parent / "SANA-model" if model.is_dir() and not model.is_symlink(): + _materialize_stage1_text_encoder(model) return if model.exists() or model.is_symlink(): raise RuntimeError(f"incomplete SANA model directory: {model}") snapshot = Path(snapshot_download(MODEL, revision=MODEL_REVISION)).resolve() _materialize_snapshot(snapshot, model) + _materialize_stage1_text_encoder(model) if __name__ == "__main__": diff --git a/families/sana_wm/tests/manifests/sana-wm-bidirectional.json b/families/sana_wm/tests/manifests/sana-wm-bidirectional.json index f998bc7c9a..53c1ccd8ff 100644 --- a/families/sana_wm/tests/manifests/sana-wm-bidirectional.json +++ b/families/sana_wm/tests/manifests/sana-wm-bidirectional.json @@ -3,7 +3,8 @@ "hf_id": "Efficient-Large-Model/SANA-WM_bidirectional", "hf_dependencies": [ { - "repo_id": "Efficient-Large-Model/gemma-2-2b-it" + "repo_id": "Efficient-Large-Model/gemma-2-2b-it", + "revision": "569d9809d0c8b6722d4d31b5a77a2ec7a400650a" } ], "bundle": "sana-wm-bidirectional.bundle", diff --git a/families/sana_wm/tests/test_e2e.py b/families/sana_wm/tests/test_e2e.py index 55102c23ff..79a9d2ef77 100644 --- a/families/sana_wm/tests/test_e2e.py +++ b/families/sana_wm/tests/test_e2e.py @@ -141,8 +141,14 @@ def _prepared_model_dir(model_dir: Path, manifest: dict, tmp_path: Path) -> Path stage1_text_encoder = Path( snapshot_download( repo_id=stage1_repo, + revision=dependencies[0].get("revision"), local_files_only=True, - allow_patterns=["config.json"], + allow_patterns=[ + "config.json", + "model*.safetensors*", + "special_tokens_map.json", + "tokenizer*", + ], ) ) prepared = tmp_path / "prepared-model" diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index 7e2efd3244..7fc401a4ce 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -54,6 +54,33 @@ def test_qualification_snapshot_is_materialized_inside_family_environment( assert prepare_environment.MODEL_REVISION == "e96271d77398def8ebb9fc595e7c0056dc625ab7" +def test_qualification_materializes_pinned_stage1_text_encoder( + monkeypatch, tmp_path: Path +) -> None: + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + for name in ("config.json", "model.safetensors.index.json", "tokenizer.json"): + (snapshot / name).write_text("{}\n", encoding="utf-8") + model = tmp_path / "model" + model.mkdir() + captured = {} + + def download(model_id, **kwargs): + captured.update(model_id=model_id, **kwargs) + return str(snapshot) + + monkeypatch.setattr(prepare_environment, "snapshot_download", download) + prepare_environment._materialize_stage1_text_encoder(model) + + destination = model / "stage1_text_encoder" + assert destination.is_dir() and not destination.is_symlink() + assert (destination / "tokenizer.json").is_file() + assert captured == { + "model_id": "Efficient-Large-Model/gemma-2-2b-it", + "revision": "569d9809d0c8b6722d4d31b5a77a2ec7a400650a", + } + + def test_qualification_environment_replaces_gui_opencv(monkeypatch) -> None: calls = [] monkeypatch.setattr( From b50b8a37f649d2dc6757b5b5df0928b234c21aa7 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sun, 20 Sep 2026 00:00:34 +0000 Subject: [PATCH 084/100] fix(benchmark): pin Wan reference attention Signed-off-by: chaofengw --- families/wan2_2_ti2v/requirements.txt | 2 +- families/wan2_2_ti2v/tests/test_reference.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/families/wan2_2_ti2v/requirements.txt b/families/wan2_2_ti2v/requirements.txt index a5ad62b1ec..bed38f0736 100644 --- a/families/wan2_2_ti2v/requirements.txt +++ b/families/wan2_2_ti2v/requirements.txt @@ -8,7 +8,7 @@ decord2==3.4.0 diffusers>=0.31.0 easydict einops==0.8.1 -flash-attn +flash-attn==2.8.3 ftfy==6.3.1 imageio[ffmpeg] librosa diff --git a/families/wan2_2_ti2v/tests/test_reference.py b/families/wan2_2_ti2v/tests/test_reference.py index 262acc177f..fdcbcea258 100644 --- a/families/wan2_2_ti2v/tests/test_reference.py +++ b/families/wan2_2_ti2v/tests/test_reference.py @@ -144,7 +144,7 @@ def test_official_reference_declares_its_real_import_dependencies() -> None: "dashscope", "diffusers>=0.31.0", "easydict", - "flash-attn", + "flash-attn==2.8.3", "imageio[ffmpeg]", "librosa", "opencv-python>=4.9.0.80", From b8c4c9efd0be9f81ee1f19f5fb7f6ece6741a951 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sun, 20 Sep 2026 00:59:34 +0000 Subject: [PATCH 085/100] fix(benchmark): use prepared SANA candidate Build the SANA-WM candidate from the family environment model directory that contains the pinned stage-one text encoder. Keep the candidate and official reference on the same prepared checkpoint layout. Signed-off-by: chaofengw --- .../tests/benchmark/sana-wm-bidirectional.yaml | 2 +- families/sana_wm/tests/test_reference_contract.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml index 63ef16e8dc..423c468a07 100644 --- a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml +++ b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: sana-wm-bidirectional -candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, revision: e96271d77398def8ebb9fc595e7c0056dc625ab7, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} +candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, revision: e96271d77398def8ebb9fc595e7c0056dc625ab7, model_directory: trtmc-reference/SANA-model, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} reference_environment: requirements: ../../requirements.txt paths: {sana_repo: trtmc-reference/Sana, sana_model: trtmc-reference/SANA-model} diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index 7fc401a4ce..ed66e2e77e 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -8,11 +8,24 @@ import numpy as np import pytest +import yaml from . import test_e2e as e2e from .benchmark import prepare_environment +def test_qualification_candidate_uses_the_prepared_family_model() -> None: + profile = yaml.safe_load( + (Path(__file__).parent / "benchmark/sana-wm-bidirectional.yaml").read_text( + encoding="utf-8" + ) + ) + + prepared_model = "trtmc-reference/SANA-model" + assert profile["candidate"]["model_directory"] == prepared_model + assert profile["reference_environment"]["paths"]["sana_model"] == prepared_model + + def test_official_source_dependencies_are_family_owned() -> None: requirements = { line.strip() From 84b9dc7c2accb5b6fd3026fc6338b1a016d85a4e Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sun, 20 Sep 2026 01:58:09 +0000 Subject: [PATCH 086/100] fix(benchmark): retain legacy image features Serialize pooled image feature values from legacy runtime tasks so user-facing Accuracy comparisons receive numerical outputs rather than shape-only summaries. Cover the contract with the native fake-family worker test. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 1 + .../native/test_benchmark_worker_e2e.cpp | 29 +++++++++++++++++++ core/runtime/tests/fake_family.cpp | 10 +++++++ 3 files changed, 40 insertions(+) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index aab8a77fad..cc26a83683 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -697,6 +697,7 @@ Json run_extract_features(trtmc::ITask& task, const Json& request, const Timing& [](const trtmc::ImageFeaturesResult& result) { return Json{{"processed_images", 1}, {"last_hidden_state_shape", result.last_hidden_state_shape}, + {"pooler_output", result.pooler_output}, {"pooler_output_shape", result.pooler_output_shape}, {"feature_elements", result.last_hidden_state.size() + result.pooler_output.size()}}; diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index d0f5debcec..15b2de6ed3 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -159,6 +159,35 @@ int main(int argc, char** argv) { std::filesystem::remove(vector_bundle); } + const auto feature_image = runtime_root / "benchmark_fake_features.ppm"; + { + std::ofstream image(feature_image, std::ios::binary); + image << "P6\n1 1\n255\n"; + image.write("ABC", 3); + } + const auto feature_bundle = runtime_root / "benchmark_fake_image_features.bundle"; + write_bundle(feature_bundle, "image_features"); + Json feature_request = request; + feature_request["case_name"] = "fake-image-features"; + feature_request["bundle"] = feature_bundle.string(); + feature_request["operation"] = "extract_features"; + feature_request["request"] = {{"image_path", feature_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << feature_request << '\n'; + } + check(std::system(command.c_str()) == 0, "legacy image feature worker completed"); + std::ifstream feature_output_file(output_path); + Json feature_result; + feature_output_file >> feature_result; + const auto& feature_summary = feature_result.at("output_summary"); + check(feature_summary.at("pooler_output") == Json::array({0.25F, 0.75F}), + "legacy image feature output retains pooled values for Accuracy comparison"); + check(feature_summary.at("pooler_output_shape") == Json::array({1, 2}), + "legacy image feature output retains pooled shape"); + std::filesystem::remove(feature_bundle); + std::filesystem::remove(feature_image); + const auto image_bundle = runtime_root / "benchmark_fake_image_generation.bundle"; write_bundle(image_bundle, "image_generation"); Json image_request = request; diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index db195081c2..adf4bd3b42 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -56,6 +56,14 @@ class FakeEmbedding final : public trtmc::IEmbedding { } }; +class FakeImageFeatures final : public trtmc::IImageFeatureExtractor { + public: + trtmc::ImageFeaturesResult extract_image_features(const float*, std::int32_t, + std::int32_t) override { + return {{1.0F, 2.0F, 3.0F, 4.0F}, {1, 2, 2}, {0.25F, 0.75F}, {1, 2}}; + } +}; + class FakeImageGeneration final : public trtmc::IImageGeneration, public trtmc::IImageEditing, public trtmc::IImageBatchGeneration { @@ -160,6 +168,8 @@ trtmc::ITask* create_fake_task(const trtmc::FamilyContext& context) { return new FakeEncoding(); if (context.reader.info().task == trtmc::IEmbedding::kTask) return new FakeEmbedding(); + if (context.reader.info().task == trtmc::IImageFeatureExtractor::kTask) + return new FakeImageFeatures(); if (context.reader.info().task == trtmc::IImageGeneration::kTask) return new FakeImageGeneration(); if (context.reader.info().task == trtmc::IAudioGeneration::kTask) From 41c009e61e1347bc5a6035bf2e20dc4eac579f4d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sun, 20 Sep 2026 02:25:01 +0000 Subject: [PATCH 087/100] fix(sana-wm): honor TensorRT install paths Pass the configured TensorRT include and library locations to the family-owned native plugin build. This allows the plugin to use the same TensorRT 11 installation as the main project in isolated benchmark environments. Signed-off-by: chaofengw --- families/sana_wm/native_plugin_builder.py | 37 ++++++++++++++----- .../sana_wm/tests/test_reference_contract.py | 13 +++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/families/sana_wm/native_plugin_builder.py b/families/sana_wm/native_plugin_builder.py index 42ef7e2e78..a29d3a64f9 100644 --- a/families/sana_wm/native_plugin_builder.py +++ b/families/sana_wm/native_plugin_builder.py @@ -6,32 +6,49 @@ from __future__ import annotations import ctypes +import os import subprocess import tempfile from pathlib import Path -from typing import Any +from typing import Any, Mapping _PLUGIN_HANDLE: Any | None = None _PLUGIN_PATH: Path | None = None -def ensure_native_plugin(*, verbose: bool = False) -> Path: - """Build a fresh plugin library and return it.""" - import torch - - source_dir = Path(__file__).with_name("native_plugins") - build_dir = Path(tempfile.mkdtemp(prefix="sana-wm-plugin-")) - output = build_dir / "libtrtmc_sana_wm_native_plugin.so" - configure = [ +def _configure_command( + source_dir: Path, + build_dir: Path, + torch_prefix: str, + environment: Mapping[str, str], +) -> list[str]: + command = [ "cmake", "-S", str(source_dir), "-B", str(build_dir), "-DCMAKE_BUILD_TYPE=Release", - f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", + f"-DCMAKE_PREFIX_PATH={torch_prefix}", ] + if include_dir := environment.get("TRT_INC_DIR"): + command.append(f"-DSANA_WM_TRT_INCLUDE_DIR={include_dir}") + if library_dir := environment.get("TRT_LIB_DIR"): + command.append(f"-DSANA_WM_TRT_LIBRARY={Path(library_dir) / 'libnvinfer.so.11'}") + return command + + +def ensure_native_plugin(*, verbose: bool = False) -> Path: + """Build a fresh plugin library and return it.""" + import torch + + source_dir = Path(__file__).with_name("native_plugins") + build_dir = Path(tempfile.mkdtemp(prefix="sana-wm-plugin-")) + output = build_dir / "libtrtmc_sana_wm_native_plugin.so" + configure = _configure_command( + source_dir, build_dir, torch.utils.cmake_prefix_path, os.environ + ) build = [ "cmake", "--build", diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index ed66e2e77e..3dcf0b83bd 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -10,6 +10,7 @@ import pytest import yaml +from .. import native_plugin_builder from . import test_e2e as e2e from .benchmark import prepare_environment @@ -26,6 +27,18 @@ def test_qualification_candidate_uses_the_prepared_family_model() -> None: assert profile["reference_environment"]["paths"]["sana_model"] == prepared_model +def test_native_plugin_uses_configured_tensorrt_prefix(tmp_path: Path) -> None: + command = native_plugin_builder._configure_command( + tmp_path / "source", + tmp_path / "build", + "/torch/cmake", + {"TRT_INC_DIR": "/tensorrt/include", "TRT_LIB_DIR": "/tensorrt/lib"}, + ) + + assert "-DSANA_WM_TRT_INCLUDE_DIR=/tensorrt/include" in command + assert "-DSANA_WM_TRT_LIBRARY=/tensorrt/lib/libnvinfer.so.11" in command + + def test_official_source_dependencies_are_family_owned() -> None: requirements = { line.strip() From e312bfe35f92bbef3f70a69da7902769b5f4f84c Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sun, 20 Sep 2026 02:54:30 +0000 Subject: [PATCH 088/100] fix(sana-wm): discover TensorRT Python library Signed-off-by: chaofengw --- families/sana_wm/native_plugin_builder.py | 34 ++++++++++++++----- .../sana_wm/tests/test_reference_contract.py | 34 +++++++++++++------ 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/families/sana_wm/native_plugin_builder.py b/families/sana_wm/native_plugin_builder.py index a29d3a64f9..321636fd99 100644 --- a/families/sana_wm/native_plugin_builder.py +++ b/families/sana_wm/native_plugin_builder.py @@ -6,11 +6,11 @@ from __future__ import annotations import ctypes -import os +import importlib.util import subprocess import tempfile from pathlib import Path -from typing import Any, Mapping +from typing import Any _PLUGIN_HANDLE: Any | None = None @@ -21,7 +21,7 @@ def _configure_command( source_dir: Path, build_dir: Path, torch_prefix: str, - environment: Mapping[str, str], + tensorrt_library: Path | None, ) -> list[str]: command = [ "cmake", @@ -32,13 +32,24 @@ def _configure_command( "-DCMAKE_BUILD_TYPE=Release", f"-DCMAKE_PREFIX_PATH={torch_prefix}", ] - if include_dir := environment.get("TRT_INC_DIR"): - command.append(f"-DSANA_WM_TRT_INCLUDE_DIR={include_dir}") - if library_dir := environment.get("TRT_LIB_DIR"): - command.append(f"-DSANA_WM_TRT_LIBRARY={Path(library_dir) / 'libnvinfer.so.11'}") + if tensorrt_library is not None: + command.append(f"-DSANA_WM_TRT_LIBRARY={tensorrt_library}") return command +def _installed_tensorrt_library() -> Path | None: + """Return the TensorRT library shipped with the active Python installation.""" + + spec = importlib.util.find_spec("tensorrt_libs") + if spec is None: + return None + for location in spec.submodule_search_locations or (): + library = Path(location) / "libnvinfer.so.11" + if library.is_file(): + return library + return None + + def ensure_native_plugin(*, verbose: bool = False) -> Path: """Build a fresh plugin library and return it.""" import torch @@ -47,7 +58,10 @@ def ensure_native_plugin(*, verbose: bool = False) -> Path: build_dir = Path(tempfile.mkdtemp(prefix="sana-wm-plugin-")) output = build_dir / "libtrtmc_sana_wm_native_plugin.so" configure = _configure_command( - source_dir, build_dir, torch.utils.cmake_prefix_path, os.environ + source_dir, + build_dir, + torch.utils.cmake_prefix_path, + _installed_tensorrt_library(), ) build = [ "cmake", @@ -57,7 +71,9 @@ def ensure_native_plugin(*, verbose: bool = False) -> Path: "trtmc_sana_wm_native_plugin", "-j2", ] - kwargs = {} if verbose else {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True} + kwargs = ( + {} if verbose else {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True} + ) try: subprocess.run(configure, check=True, **kwargs) subprocess.run(build, check=True, **kwargs) diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index 3dcf0b83bd..2eb0c7e513 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -17,9 +17,7 @@ def test_qualification_candidate_uses_the_prepared_family_model() -> None: profile = yaml.safe_load( - (Path(__file__).parent / "benchmark/sana-wm-bidirectional.yaml").read_text( - encoding="utf-8" - ) + (Path(__file__).parent / "benchmark/sana-wm-bidirectional.yaml").read_text(encoding="utf-8") ) prepared_model = "trtmc-reference/SANA-model" @@ -27,16 +25,34 @@ def test_qualification_candidate_uses_the_prepared_family_model() -> None: assert profile["reference_environment"]["paths"]["sana_model"] == prepared_model -def test_native_plugin_uses_configured_tensorrt_prefix(tmp_path: Path) -> None: +def test_native_plugin_uses_installed_tensorrt_library(tmp_path: Path) -> None: + tensorrt_library = tmp_path / "tensorrt/libnvinfer.so.11" command = native_plugin_builder._configure_command( tmp_path / "source", tmp_path / "build", "/torch/cmake", - {"TRT_INC_DIR": "/tensorrt/include", "TRT_LIB_DIR": "/tensorrt/lib"}, + tensorrt_library, + ) + + assert f"-DSANA_WM_TRT_LIBRARY={tensorrt_library}" in command + + +def test_native_plugin_discovers_tensorrt_python_library( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + package = tmp_path / "tensorrt_libs" + package.mkdir() + library = package / "libnvinfer.so.11" + library.touch() + spec = native_plugin_builder.importlib.util.spec_from_file_location( + "tensorrt_libs", + package / "__init__.py", + submodule_search_locations=[str(package)], ) + monkeypatch.setattr(native_plugin_builder.importlib.util, "find_spec", lambda _name: spec) - assert "-DSANA_WM_TRT_INCLUDE_DIR=/tensorrt/include" in command - assert "-DSANA_WM_TRT_LIBRARY=/tensorrt/lib/libnvinfer.so.11" in command + assert native_plugin_builder._installed_tensorrt_library() == library def test_official_source_dependencies_are_family_owned() -> None: @@ -80,9 +96,7 @@ def test_qualification_snapshot_is_materialized_inside_family_environment( assert prepare_environment.MODEL_REVISION == "e96271d77398def8ebb9fc595e7c0056dc625ab7" -def test_qualification_materializes_pinned_stage1_text_encoder( - monkeypatch, tmp_path: Path -) -> None: +def test_qualification_materializes_pinned_stage1_text_encoder(monkeypatch, tmp_path: Path) -> None: snapshot = tmp_path / "snapshot" snapshot.mkdir() for name in ("config.json", "model.safetensors.index.json", "tokenizer.json"): From bfc92727d2c978bbc940180b923b9665eaef0acd Mon Sep 17 00:00:00 2001 From: chaofengw Date: Sun, 20 Sep 2026 11:11:02 +0000 Subject: [PATCH 089/100] fix(benchmark): restore report JSON filename Write the aggregate qualification result to report.json again and keep the HTML and terminal links consistent. Cover the filename contract so summary.json is not reintroduced accidentally. Signed-off-by: chaofengw --- tools/model_benchmark.py | 6 +++--- tools/tests/test_model_benchmark.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tools/model_benchmark.py b/tools/model_benchmark.py index d2e8708bb0..bc2ca63097 100644 --- a/tools/model_benchmark.py +++ b/tools/model_benchmark.py @@ -126,13 +126,13 @@ def _run(cases: Sequence[QualificationCase], arguments: argparse.Namespace) -> i "cases": results, } _write_summary(context.artifacts, summary) - print(f"JSON: {context.artifacts / 'summary.json'}") + print(f"JSON: {context.artifacts / 'report.json'}") print(f"HTML: {context.artifacts / 'report.html'}") return 0 if summary["status"] == "passed" else 1 def _write_summary(output: Path, summary: Mapping[str, Any]) -> None: - (output / "summary.json").write_text( + (output / "report.json").write_text( json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) rows = [] @@ -148,7 +148,7 @@ def _write_summary(output: Path, summary: Mapping[str, Any]) -> None: document = """TRTMC model benchmark

Internal model benchmark

Status: {status}

-{rows}
ModelKindCaseStatus

summary.json

+{rows}

report.json

""".format(status=html.escape(str(summary["status"])), rows="".join(rows)) (output / "report.html").write_text(document, encoding="utf-8") diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 83f254e7fc..234b736a2e 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -14,6 +14,7 @@ import pytest +from tools import model_benchmark from apps.benchmark.performance.baselines import hf_transformers, task_reference from apps.benchmark.performance.baselines.timing_contracts import timing_contract from families.personaplex.tests.benchmark import prepare_environment as personaplex_environment @@ -40,6 +41,29 @@ REPOSITORY = Path(__file__).resolve().parents[2] +def test_qualification_summary_is_written_as_report_json(tmp_path: Path) -> None: + summary = { + "schema_version": "trtmc.qualification-summary/v1", + "status": "passed", + "cases": [ + { + "model": "example-model", + "kind": "accuracy", + "case": "example-model/accuracy/parity", + "status": "passed", + } + ], + } + + model_benchmark._write_summary(tmp_path, summary) + + assert json.loads((tmp_path / "report.json").read_text(encoding="utf-8")) == summary + assert not (tmp_path / "summary.json").exists() + html_report = (tmp_path / "report.html").read_text(encoding="utf-8") + assert 'href="report.json"' in html_report + assert "summary.json" not in html_report + + def test_family_configs_auto_discover_without_a_central_model_registry() -> None: cases = discover(REPOSITORY) assert cases From 05c79a3a39f363e8c9098867c6d8380b2ffee1a3 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 21 Sep 2026 03:55:26 +0000 Subject: [PATCH 090/100] fix(benchmark): gate detection on COCO ground truth Replace Imagenette output parity with a deterministic 100-image COCO val2017 subset. Evaluate the candidate and family reference against real annotations with AP50 and AP50:95 while keeping each family's label space explicit. Add a reproducible dataset preparation tool and regression coverage. Signed-off-by: chaofengw --- .../detr/tests/benchmark/detr-resnet-50.yaml | 13 +- families/yolo11/tests/benchmark/yolo11n.yaml | 13 +- .../yolov10/tests/benchmark/yolov10n.yaml | 13 +- .../yolov10/tests/benchmark/yolov10s.yaml | 13 +- .../yolov10/tests/benchmark/yolov10x.yaml | 13 +- families/yolov5/tests/benchmark/yolov5n.yaml | 13 +- families/yolov8/tests/benchmark/yolov8n.yaml | 13 +- tools/benchmark_qualification/accuracy.py | 273 ++++++++++++------ .../benchmarks/coco2017_object_detection.yaml | 20 ++ .../imagenette_detection_parity.yaml | 20 -- tools/prepare_coco_detection_dataset.py | 201 +++++++++++++ tools/tests/test_architecture.py | 3 +- tools/tests/test_model_benchmark.py | 101 +++++-- 13 files changed, 538 insertions(+), 171 deletions(-) create mode 100644 tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml delete mode 100644 tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml create mode 100644 tools/prepare_coco_detection_dataset.py diff --git a/families/detr/tests/benchmark/detr-resnet-50.yaml b/families/detr/tests/benchmark/detr-resnet-50.yaml index 211aec65a1..1d7b65caa4 100644 --- a/families/detr/tests/benchmark/detr-resnet-50.yaml +++ b/families/detr/tests/benchmark/detr-resnet-50.yaml @@ -19,19 +19,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-category-id request: &detection-request score_threshold: 0.5 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.5 - max_score_abs_error: 0.05 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/families/yolo11/tests/benchmark/yolo11n.yaml b/families/yolo11/tests/benchmark/yolo11n.yaml index be9e997b15..359f9cc94b 100644 --- a/families/yolo11/tests/benchmark/yolo11n.yaml +++ b/families/yolo11/tests/benchmark/yolo11n.yaml @@ -17,19 +17,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 request: &detection-request score_threshold: 0.25 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.9 - max_score_abs_error: 0.005 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/families/yolov10/tests/benchmark/yolov10n.yaml b/families/yolov10/tests/benchmark/yolov10n.yaml index 4b631bd899..8a5fc4bdc5 100644 --- a/families/yolov10/tests/benchmark/yolov10n.yaml +++ b/families/yolov10/tests/benchmark/yolov10n.yaml @@ -17,19 +17,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 request: &detection-request score_threshold: 0.25 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.9 - max_score_abs_error: 0.005 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/families/yolov10/tests/benchmark/yolov10s.yaml b/families/yolov10/tests/benchmark/yolov10s.yaml index 141c6d9bab..2855a485b4 100644 --- a/families/yolov10/tests/benchmark/yolov10s.yaml +++ b/families/yolov10/tests/benchmark/yolov10s.yaml @@ -17,19 +17,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 request: &detection-request score_threshold: 0.25 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.9 - max_score_abs_error: 0.005 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/families/yolov10/tests/benchmark/yolov10x.yaml b/families/yolov10/tests/benchmark/yolov10x.yaml index 1f7a4c9995..bd7b843435 100644 --- a/families/yolov10/tests/benchmark/yolov10x.yaml +++ b/families/yolov10/tests/benchmark/yolov10x.yaml @@ -17,19 +17,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 request: &detection-request score_threshold: 0.25 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.9 - max_score_abs_error: 0.005 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/families/yolov5/tests/benchmark/yolov5n.yaml b/families/yolov5/tests/benchmark/yolov5n.yaml index 31a201cccf..4e3906de0d 100644 --- a/families/yolov5/tests/benchmark/yolov5n.yaml +++ b/families/yolov5/tests/benchmark/yolov5n.yaml @@ -17,19 +17,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 request: &detection-request score_threshold: 0.25 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.9 - max_score_abs_error: 0.005 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/families/yolov8/tests/benchmark/yolov8n.yaml b/families/yolov8/tests/benchmark/yolov8n.yaml index eb6478fbf4..d7a1ed44ef 100644 --- a/families/yolov8/tests/benchmark/yolov8n.yaml +++ b/families/yolov8/tests/benchmark/yolov8n.yaml @@ -17,19 +17,18 @@ reference_environment: requirements: ../../requirements.txt accuracy: - - name: imagenette-detection-parity - benchmark: imagenette_detection_parity - samples: 10 + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 request: &detection-request score_threshold: 0.25 reference: command: reference.py precision: fp32 gate: - min_box_iou: 0.9 - max_score_abs_error: 0.005 - min_detection_match_rate: 1.0 - min_sample_pass_rate: 0.9 + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 performance: - name: detect diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 82bf1b873d..066c6346a4 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -6,6 +6,7 @@ from __future__ import annotations from array import array +from collections import defaultdict import csv from itertools import permutations import json @@ -66,8 +67,8 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, result = _speech_transcription_parity(case, context, dataset, output) elif metric_name == "image_feature_knn_parity": result = _image_feature_knn_parity(case, context, dataset, output) - elif metric_name == "object_detection_parity": - result = _object_detection_parity(case, context, dataset, output) + elif metric_name == "coco_object_detection_accuracy": + result = _coco_object_detection_accuracy(case, context, dataset, output) elif metric_name == "prompted_segmentation_parity": result = _prompted_segmentation_parity(case, context, dataset, output) elif metric_name == "text_prompted_instance_segmentation_parity": @@ -1595,35 +1596,165 @@ def _detections(value: Mapping[str, Any], label: str) -> list[dict[str, Any]]: return detections -def _match_detections( - candidate: Sequence[Mapping[str, Any]], reference: Sequence[Mapping[str, Any]] -) -> list[dict[str, Any]]: - matched_reference: set[int] = set() - matches = [] - for detection in sorted(candidate, key=lambda item: -float(item["score"])): - choices = [ - (_box_iou(detection["box"], expected["box"]), index) - for index, expected in enumerate(reference) - if index not in matched_reference - and int(detection["class_id"]) == int(expected["class_id"]) - ] - if not choices: - continue - iou, index = max(choices) - matched_reference.add(index) - matches.append( +def _coco_detection_samples( + dataset: Dataset, sample_limit: int +) -> tuple[list[dict[str, Any]], str]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("COCO object-detection dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError( + f"COCO object-detection dataset requires at least {sample_limit} requests" + ) + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"COCO object-detection request {index} is invalid") + relative = request.get("image") + annotations = request.get("annotations") + if not isinstance(relative, str) or not relative: + raise QualificationError(f"COCO object-detection request {index} has no image") + if not isinstance(annotations, list) or not annotations: + raise QualificationError( + f"COCO object-detection request {index} has no ground-truth annotations" + ) + image = (root / relative).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError( + f"COCO object-detection request {index} image is unavailable: {image}" + ) + ground_truth = [] + for annotation_index, annotation in enumerate(annotations): + if not isinstance(annotation, Mapping): + raise QualificationError( + f"COCO object-detection annotation {index}:{annotation_index} is invalid" + ) + box = annotation.get("bbox_xyxy") + category_id = annotation.get("category_id") + category_index = annotation.get("category_index") + if ( + not isinstance(box, list) + or len(box) != 4 + or any(not isinstance(value, (int, float)) for value in box) + or isinstance(category_id, bool) + or not isinstance(category_id, int) + or isinstance(category_index, bool) + or not isinstance(category_index, int) + ): + raise QualificationError( + f"COCO object-detection annotation {index}:{annotation_index} is invalid" + ) + numeric_box = [float(value) for value in box] + if ( + not all(math.isfinite(value) for value in numeric_box) + or numeric_box[2] <= numeric_box[0] + or numeric_box[3] <= numeric_box[1] + ): + raise QualificationError( + f"COCO object-detection annotation {index}:{annotation_index} has an invalid box" + ) + ground_truth.append( + { + "box": numeric_box, + "category_id": category_id, + "category_index": category_index, + } + ) + selected.append( { - "class_id": int(detection["class_id"]), - "iou": iou, - "score_abs_error": abs( - float(detection["score"]) - float(reference[index]["score"]) - ), + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + "annotations": ground_truth, } ) - return matches + sampling = payload.get("sampling") + return selected, str(sampling) if isinstance(sampling, str) else "fixed manifest order" + + +def _coco_ap_at_iou( + selected: Sequence[Mapping[str, Any]], + outputs: Sequence[Mapping[str, Any]], + label_field: str, + iou_threshold: float, +) -> float: + ground_truth: dict[int, dict[str, list[Sequence[float]]]] = {} + predictions: dict[int, list[tuple[float, str, Sequence[float]]]] = {} + for sample, output in zip(selected, outputs, strict=True): + sample_id = str(sample["sample_id"]) + for annotation in sample["annotations"]: + class_id = int(annotation[label_field]) + ground_truth.setdefault(class_id, {}).setdefault(sample_id, []).append( + annotation["box"] + ) + for detection in sorted( + _detections(output, "object-detection output"), + key=lambda value: -float(value["score"]), + )[:100]: + predictions.setdefault(int(detection["class_id"]), []).append( + (float(detection["score"]), sample_id, detection["box"]) + ) + category_aps = [] + for class_id, sample_boxes in ground_truth.items(): + total_ground_truth = sum(len(boxes) for boxes in sample_boxes.values()) + matched: dict[str, set[int]] = defaultdict(set) + true_positives = false_positives = 0 + precisions = [] + recalls = [] + for _, sample_id, box in sorted(predictions.get(class_id, []), key=lambda value: -value[0]): + candidates = [ + (_box_iou(box, expected), index) + for index, expected in enumerate(sample_boxes.get(sample_id, [])) + if index not in matched[sample_id] + ] + if candidates and max(candidates)[0] >= iou_threshold: + _, matched_index = max(candidates) + matched[sample_id].add(matched_index) + true_positives += 1 + else: + false_positives += 1 + precisions.append(true_positives / (true_positives + false_positives)) + recalls.append(true_positives / total_ground_truth) + category_aps.append( + sum( + max( + ( + precision + for precision, recall in zip(precisions, recalls, strict=True) + if recall >= point + ), + default=0.0, + ) + for point in (index / 100.0 for index in range(101)) + ) + / 101.0 + ) + if not category_aps: + raise QualificationError("COCO object-detection subset has no ground truth") + return sum(category_aps) / len(category_aps) + + +def _coco_detection_metrics( + selected: Sequence[Mapping[str, Any]], + outputs: Sequence[Mapping[str, Any]], + label_field: str, +) -> dict[str, float]: + if len(selected) != len(outputs): + raise QualificationError("COCO object-detection output count differs from the dataset") + thresholds = [0.5 + index * 0.05 for index in range(10)] + average_precisions = [ + _coco_ap_at_iou(selected, outputs, label_field, threshold) for threshold in thresholds + ] + return { + "map_50_95": sum(average_precisions) / len(average_precisions), + "map_50": average_precisions[0], + } -def _object_detection_parity( + +def _coco_object_detection_accuracy( case: QualificationCase, context: RuntimeContext, dataset: Dataset, @@ -1631,11 +1762,11 @@ def _object_detection_parity( ) -> dict[str, Any]: configured = case.values sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") - selected = _image_parity_samples(dataset, sample_limit, "object-detection") + selected, sampling = _coco_detection_samples(dataset, sample_limit) expected = _family_image_reference(case, context, output, selected) request = configured.get("request", {}) if not isinstance(request, Mapping): - raise QualificationError("object-detection request must be an object") + raise QualificationError("COCO object-detection request must be an object") candidate_requests = [ { "sample_id": sample["sample_id"], @@ -1646,52 +1777,26 @@ def _object_detection_parity( actual, bundle = _candidate_outputs(case, context, output, "detect", candidate_requests) gate = configured.get("gate", {}) if not isinstance(gate, Mapping): - raise QualificationError("object-detection gate must be an object") - minimum_iou = float(gate.get("min_box_iou", 0.5)) - maximum_score_error = float(gate.get("max_score_abs_error", 0.05)) - minimum_match_rate = float(gate.get("min_detection_match_rate", 1.0)) - minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) - if not all( - 0.0 <= value <= 1.0 for value in (minimum_iou, minimum_match_rate, minimum_sample_rate) - ): - raise QualificationError("object-detection rates and IoU must be in [0, 1]") - if maximum_score_error < 0.0 or not math.isfinite(maximum_score_error): - raise QualificationError("max_score_abs_error must be finite and nonnegative") - - rows = [] - total_matches = total_detections = 0 - for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): - candidate_detections = _detections(candidate_sample, "candidate") - reference_detections = _detections(reference_sample, "reference") - matches = _match_detections(candidate_detections, reference_detections) - denominator = max(len(candidate_detections), len(reference_detections), 1) - match_rate = len(matches) / denominator - minimum_sample_iou = min((match["iou"] for match in matches), default=1.0) - maximum_sample_score_error = max( - (match["score_abs_error"] for match in matches), default=0.0 - ) - passed = ( - match_rate >= minimum_match_rate - and minimum_sample_iou >= minimum_iou - and maximum_sample_score_error <= maximum_score_error - ) - rows.append( - { - "sample_id": sample["sample_id"], - "passed": passed, - "candidate_detections": len(candidate_detections), - "reference_detections": len(reference_detections), - "matched_detections": len(matches), - "detection_match_rate": match_rate, - "min_box_iou": minimum_sample_iou, - "max_score_abs_error": maximum_sample_score_error, - } + raise QualificationError("COCO object-detection gate must be an object") + label_spaces = { + "coco-category-id": "category_id", + "coco-contiguous-80": "category_index", + } + label_space = configured.get("label_space") + if label_space not in label_spaces: + raise QualificationError( + "COCO object-detection label_space must be coco-category-id or coco-contiguous-80" ) - total_matches += len(matches) - total_detections += denominator - sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) - detection_match_rate = total_matches / total_detections - passed = sample_pass_rate >= minimum_sample_rate + map_50_95_limit = float(gate.get("max_map_50_95_drop", 0.02)) + map_50_limit = float(gate.get("max_map_50_drop", 0.02)) + if not all(math.isfinite(value) and value >= 0.0 for value in (map_50_95_limit, map_50_limit)): + raise QualificationError("COCO object-detection mAP drops must be finite and nonnegative") + label_field = label_spaces[str(label_space)] + candidate_metrics = _coco_detection_metrics(selected, actual, label_field) + reference_metrics = _coco_detection_metrics(selected, expected, label_field) + map_50_95_drop = reference_metrics["map_50_95"] - candidate_metrics["map_50_95"] + map_50_drop = reference_metrics["map_50"] - candidate_metrics["map_50"] + passed = map_50_95_drop <= map_50_95_limit and map_50_drop <= map_50_limit return { "schema_version": "trtmc.qualification-result/v1", "case": case.id, @@ -1700,21 +1805,21 @@ def _object_detection_parity( "model": case.model, "benchmark": case.benchmark, "bundle": str(bundle), - "dataset": dataset.receipt(), + "dataset": {**dataset.receipt(), "sampling": sampling}, "metrics": { - "samples": len(rows), - "sample_pass_rate": sample_pass_rate, - "detection_match_rate": detection_match_rate, - "min_box_iou": min(row["min_box_iou"] for row in rows), - "max_score_abs_error": max(row["max_score_abs_error"] for row in rows), + "samples": len(selected), + "label_space": label_space, + "candidate_map_50_95": candidate_metrics["map_50_95"], + "reference_map_50_95": reference_metrics["map_50_95"], + "map_50_95_drop": map_50_95_drop, + "candidate_map_50": candidate_metrics["map_50"], + "reference_map_50": reference_metrics["map_50"], + "map_50_drop": map_50_drop, }, "gate": { - "min_box_iou": minimum_iou, - "max_score_abs_error": maximum_score_error, - "min_detection_match_rate": minimum_match_rate, - "min_sample_pass_rate": minimum_sample_rate, + "max_map_50_95_drop": map_50_95_limit, + "max_map_50_drop": map_50_limit, }, - "samples": rows, } diff --git a/tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml b/tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml new file mode 100644 index 0000000000..3d692db04a --- /dev/null +++ b/tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: coco2017-object-detection + version: val2017-category-balanced-100 + sha256: 5fca4f7e831e99cc721fc9340bf76dc48086548d4fb4a7c84512a18b91edae20 + source: + mode: manual + path: COCO2017_object_detection/coco2017_object_detection.json + instructions: tools/prepare_coco_detection_dataset.py + license: COCO images retain their source licenses; annotations are CC BY 4.0 +selection: + method: fixed-manifest-order +input: + adapter: image_path +metric: + name: coco_object_detection_accuracy diff --git a/tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml b/tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml deleted file mode 100644 index 61c379ee10..0000000000 --- a/tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Internal reusable benchmark definition; it does not register a model. -kind: accuracy -dataset: - id: imagenette-validation - version: imagenette2-320-task-eval - sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 - source: - mode: manual - path: Imagenette/imagenette2-320_task_eval.json - instructions: https://github.com/fastai/imagenette - license: Apache-2.0 -selection: - method: first -input: - adapter: image_path -metric: - name: object_detection_parity diff --git a/tools/prepare_coco_detection_dataset.py b/tools/prepare_coco_detection_dataset.py new file mode 100644 index 0000000000..b976d9cc41 --- /dev/null +++ b/tools/prepare_coco_detection_dataset.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare a deterministic COCO val2017 object-detection subset.""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +import json +import shutil +from pathlib import Path +from typing import Any, Mapping, Sequence + + +def _positive_int(raw: str) -> int: + value = int(raw) + if value < 1: + raise argparse.ArgumentTypeError("value must be positive") + return value + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--coco-root", + required=True, + type=Path, + help="Directory containing annotations/instances_val2017.json and val2017/.", + ) + parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument("--limit", default=100, type=_positive_int) + return parser + + +def _objects(raw: Any, label: str) -> list[dict[str, Any]]: + if not isinstance(raw, list) or not all(isinstance(value, Mapping) for value in raw): + raise ValueError(f"COCO {label} must be a list of objects") + return [dict(value) for value in raw] + + +def _valid_annotation(annotation: Mapping[str, Any]) -> bool: + bbox = annotation.get("bbox") + return ( + not bool(annotation.get("iscrowd", 0)) + and isinstance(bbox, list) + and len(bbox) == 4 + and float(bbox[2]) > 0.0 + and float(bbox[3]) > 0.0 + and float(annotation.get("area", float(bbox[2]) * float(bbox[3]))) > 0.0 + ) + + +def _select_images( + images: Sequence[Mapping[str, Any]], + annotations: Sequence[Mapping[str, Any]], + category_ids: Sequence[int], + limit: int, +) -> list[int]: + image_by_id = {int(image["id"]): image for image in images} + by_category: dict[int, list[int]] = defaultdict(list) + seen_pairs: set[tuple[int, int]] = set() + for annotation in sorted(annotations, key=lambda value: int(value["id"])): + if not _valid_annotation(annotation): + continue + image_id = int(annotation["image_id"]) + category_id = int(annotation["category_id"]) + image = image_by_id.get(image_id) + if image is None or category_id not in category_ids: + continue + bbox = annotation["bbox"] + image_area = float(image["width"]) * float(image["height"]) + box_area = float(annotation.get("area", float(bbox[2]) * float(bbox[3]))) + pair = (category_id, image_id) + if box_area >= image_area * 0.01 and pair not in seen_pairs: + by_category[category_id].append(image_id) + seen_pairs.add(pair) + + selected: list[int] = [] + selected_set: set[int] = set() + offset = 0 + while len(selected) < limit: + made_progress = False + for category_id in category_ids: + rows = by_category.get(category_id, []) + if offset >= len(rows): + continue + made_progress = True + image_id = rows[offset] + if image_id in selected_set: + continue + selected.append(image_id) + selected_set.add(image_id) + if len(selected) == limit: + return selected + if not made_progress: + break + offset += 1 + raise ValueError(f"COCO annotations provide only {len(selected)} eligible unique images") + + +def prepare(coco_root: Path, output_root: Path, limit: int) -> Path: + annotation_path = coco_root / "annotations" / "instances_val2017.json" + image_root = coco_root / "val2017" + payload = json.loads(annotation_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError("COCO annotation file must contain an object") + images = _objects(payload.get("images"), "images") + annotations = _objects(payload.get("annotations"), "annotations") + categories = sorted( + _objects(payload.get("categories"), "categories"), key=lambda value: int(value["id"]) + ) + category_ids = [int(category["id"]) for category in categories] + category_indices = {category_id: index for index, category_id in enumerate(category_ids)} + selected_ids = _select_images(images, annotations, category_ids, limit) + selected_set = set(selected_ids) + image_by_id = {int(image["id"]): image for image in images} + annotations_by_image: dict[int, list[dict[str, Any]]] = defaultdict(list) + for annotation in annotations: + image_id = int(annotation["image_id"]) + if image_id not in selected_set or not _valid_annotation(annotation): + continue + x, y, width, height = (float(value) for value in annotation["bbox"]) + category_id = int(annotation["category_id"]) + annotations_by_image[image_id].append( + { + "id": int(annotation["id"]), + "category_id": category_id, + "category_index": category_indices[category_id], + "bbox_xyxy": [x, y, x + width, y + height], + "area": float(annotation.get("area", width * height)), + } + ) + + output_dir = output_root / "COCO2017_object_detection" + requests = [] + for image_id in selected_ids: + image = image_by_id[image_id] + filename = str(image["file_name"]) + relative = Path("images") / filename + source = image_root / filename + if not source.is_file(): + raise FileNotFoundError(f"COCO image is unavailable: {source}") + destination = output_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + requests.append( + { + "id": f"coco2017_{image_id:012d}", + "image": relative.as_posix(), + "image_id": image_id, + "width": int(image["width"]), + "height": int(image["height"]), + "annotations": sorted( + annotations_by_image[image_id], key=lambda value: int(value["id"]) + ), + } + ) + + manifest = { + "dataset": "COCO 2017 validation object detection", + "sources": [ + "http://images.cocodataset.org/zips/val2017.zip", + "http://images.cocodataset.org/annotations/annotations_trainval2017.zip", + ], + "sampling": ( + "deterministic category-balanced round-robin over unique images; " + "each selected category has a non-crowd instance covering at least 1% of the image" + ), + "label_spaces": { + "coco-category-id": "category_id", + "coco-contiguous-80": "category_index", + }, + "categories": [ + { + "id": int(category["id"]), + "index": category_indices[int(category["id"])], + "name": str(category["name"]), + } + for category in categories + ], + "requests": requests, + } + output = output_dir / "coco2017_object_detection.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return output + + +def main() -> int: + arguments = _parser().parse_args() + output = prepare( + arguments.coco_root.resolve(), arguments.output_root.resolve(), arguments.limit + ) + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 16aef8e646..920a9de8b7 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -525,7 +525,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", "tools/benchmark_qualification/benchmarks/image_features_performance.yaml", "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_detection_parity.yaml", + "tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml", "tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", @@ -565,6 +565,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/benchmark_qualification/references/speech_io.py", "tools/benchmark_qualification/references/timm_image_classification.py", "tools/benchmark_qualification/runtime.py", + "tools/prepare_coco_detection_dataset.py", "tools/ci/__init__.py", "tools/ci/__main__.py", "tools/ci/container.py", diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 234b736a2e..746256c030 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -14,7 +14,7 @@ import pytest -from tools import model_benchmark +from tools import model_benchmark, prepare_coco_detection_dataset from apps.benchmark.performance.baselines import hf_transformers, task_reference from apps.benchmark.performance.baselines.timing_contracts import timing_contract from families.personaplex.tests.benchmark import prepare_environment as personaplex_environment @@ -607,16 +607,31 @@ def test_robot_action_accuracy_compares_complete_action_chunk( assert result["metrics"]["action_max_abs_error"] == pytest.approx(0.0) -def test_object_detection_accuracy_matches_classes_boxes_and_scores( - tmp_path: Path, monkeypatch -) -> None: - dataset_root = tmp_path / "Imagenette" +def test_coco_object_detection_accuracy_uses_ground_truth_map(tmp_path: Path, monkeypatch) -> None: + dataset_root = tmp_path / "COCO2017_object_detection" dataset_root.mkdir() image = dataset_root / "image.jpeg" image.write_bytes(b"fixture") dataset_path = dataset_root / "manifest.json" dataset_path.write_text( - json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), + json.dumps( + { + "sampling": "test fixture", + "requests": [ + { + "id": "image", + "image": "image.jpeg", + "annotations": [ + { + "bbox_xyxy": [10.0, 20.0, 50.0, 70.0], + "category_id": 3, + "category_index": 2, + } + ], + } + ], + } + ), encoding="utf-8", ) profile = tmp_path / "families/detr/tests/benchmark/example.yaml" @@ -627,8 +642,8 @@ def test_object_detection_accuracy_matches_classes_boxes_and_scores( kind="accuracy", model="example", family="detr", - name="detection-parity", - benchmark="imagenette_detection_parity", + name="coco2017-object-detection", + benchmark="coco2017_object_detection", candidate={ "family": "detr", "checkpoint": "facebook/example", @@ -638,19 +653,18 @@ def test_object_detection_accuracy_matches_classes_boxes_and_scores( }, values={ "samples": 1, + "label_space": "coco-category-id", "request": {"score_threshold": 0.5}, "reference": {"command": "reference.py", "precision": "fp32"}, "gate": { - "min_box_iou": 0.9, - "max_score_abs_error": 0.05, - "min_detection_match_rate": 1.0, - "min_sample_pass_rate": 1.0, + "max_map_50_95_drop": 0.02, + "max_map_50_drop": 0.02, }, }, source=profile, reference_requirements=None, ) - dataset = Dataset("imagenette-validation", dataset_path, "provided", "digest") + dataset = Dataset("coco2017-object-detection", dataset_path, "provided", "digest") context = RuntimeContext( repository=REPOSITORY, artifacts=tmp_path / "artifacts", @@ -689,7 +703,7 @@ def reference(command, *_args, **_kwargs): monkeypatch.setattr( qualification_accuracy, "load_benchmark", - lambda *_args: {"metric": {"name": "object_detection_parity"}}, + lambda *_args: {"metric": {"name": "coco_object_detection_accuracy"}}, ) monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) monkeypatch.setattr( @@ -702,7 +716,7 @@ def reference(command, *_args, **_kwargs): lambda *_args: ( [ { - "boxes": [10.5, 20.0, 50.0, 70.0], + "boxes": [10.0, 20.0, 50.0, 70.0], "scores": [0.89], "class_ids": [3], } @@ -714,8 +728,61 @@ def reference(command, *_args, **_kwargs): result = qualification_accuracy.run_accuracy(case, context) assert result["status"] == "passed" - assert result["metrics"]["detection_match_rate"] == 1.0 - assert result["metrics"]["min_box_iou"] >= 0.9 + assert result["metrics"]["candidate_map_50_95"] == pytest.approx(1.0) + assert result["metrics"]["reference_map_50_95"] == pytest.approx(1.0) + assert result["metrics"]["map_50_95_drop"] == pytest.approx(0.0) + + +def test_prepare_coco_detection_dataset_is_balanced_and_keeps_real_boxes( + tmp_path: Path, +) -> None: + coco_root = tmp_path / "source" + (coco_root / "annotations").mkdir(parents=True) + (coco_root / "val2017").mkdir() + images = [ + {"id": 11, "file_name": "000000000011.jpg", "width": 100, "height": 100}, + {"id": 22, "file_name": "000000000022.jpg", "width": 100, "height": 100}, + ] + annotations = [ + { + "id": 101, + "image_id": 11, + "category_id": 1, + "bbox": [10, 20, 30, 40], + "area": 1200, + "iscrowd": 0, + }, + { + "id": 202, + "image_id": 22, + "category_id": 3, + "bbox": [1, 2, 10, 20], + "area": 200, + "iscrowd": 0, + }, + ] + categories = [{"id": 1, "name": "person"}, {"id": 3, "name": "car"}] + (coco_root / "annotations" / "instances_val2017.json").write_text( + json.dumps({"images": images, "annotations": annotations, "categories": categories}), + encoding="utf-8", + ) + for image in images: + (coco_root / "val2017" / image["file_name"]).write_bytes(b"fixture") + + output = prepare_coco_detection_dataset.prepare(coco_root, tmp_path / "output", 2) + manifest = json.loads(output.read_text(encoding="utf-8")) + + assert [request["image_id"] for request in manifest["requests"]] == [11, 22] + assert manifest["requests"][0]["annotations"] == [ + { + "id": 101, + "category_id": 1, + "category_index": 0, + "bbox_xyxy": [10.0, 20.0, 40.0, 60.0], + "area": 1200.0, + } + ] + assert manifest["requests"][1]["annotations"][0]["category_index"] == 1 def test_prompted_segmentation_masks_match_independent_of_order() -> None: From c7234e8bce7c5cbc3b08ddf0ca009d7247bd6b4f Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 21 Sep 2026 06:59:48 +0000 Subject: [PATCH 091/100] fix(benchmark): diversify qualification samples Keep the runtime budget at ten samples per case while selecting fixed representative MMLU subjects, LibriSpeech speakers, and OCRBench task types. Record the selected indices in each result for reproducibility. Signed-off-by: chaofengw --- tools/benchmark_qualification/accuracy.py | 73 +++++++++++++--- .../benchmarks/librispeech_transcription.yaml | 15 +++- .../benchmarks/mmlu_continuation.yaml | 15 +++- .../benchmarks/ocrbench_v2_parity.yaml | 15 +++- tools/tests/test_model_benchmark.py | 85 ++++++++++++++----- 5 files changed, 167 insertions(+), 36 deletions(-) diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 066c6346a4..90edcddb4f 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -54,7 +54,7 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, else: dataset = resolve_dataset(definition, context, case.source) if metric_name == "exact_token_ids": - result = _text_generation_parity(case, context, dataset, output) + result = _text_generation_parity(case, context, definition, dataset, output) elif metric_name == "embedding_vector_parity": result = _encoder_embedding_parity(case, context, dataset, output) elif metric_name == "reranking_score_parity": @@ -64,7 +64,7 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, elif metric_name == "image_classification_top1_parity": result = _image_classification_parity(case, context, dataset, output) elif metric_name == "speech_transcription_wer_parity": - result = _speech_transcription_parity(case, context, dataset, output) + result = _speech_transcription_parity(case, context, definition, dataset, output) elif metric_name == "image_feature_knn_parity": result = _image_feature_knn_parity(case, context, dataset, output) elif metric_name == "coco_object_detection_accuracy": @@ -78,7 +78,7 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, elif metric_name == "vision_language_text_parity": result = _vision_language_text_parity(case, context, dataset, output) elif metric_name == "ocr_text_parity": - result = _ocr_text_parity(case, context, dataset, output) + result = _ocr_text_parity(case, context, definition, dataset, output) elif metric_name == "localization_text_parity": result = _localization_text_parity(case, context, dataset, output) elif metric_name == "stereo_disparity_parity": @@ -620,6 +620,7 @@ def _stereo_disparity_parity( def _text_generation_parity( case: QualificationCase, context: RuntimeContext, + definition: Mapping[str, Any], dataset: Dataset, output: Path, ) -> dict[str, Any]: @@ -629,8 +630,11 @@ def _text_generation_parity( requests = payload.get("requests") if isinstance(payload, Mapping) else None if not isinstance(requests, list) or not requests: raise QualificationError("text-generation dataset must contain a non-empty requests list") + selected_requests, selection = _select_dataset_rows( + requests, definition, sample_limit, "text-generation" + ) selected = [] - for index, request in enumerate(requests[:sample_limit]): + for index, request in enumerate(selected_requests): if not isinstance(request, Mapping): raise QualificationError(f"text-generation request {index} must be an object") selected.append( @@ -766,7 +770,7 @@ def _text_generation_parity( "model": case.model, "benchmark": case.benchmark, "bundle": str(bundle), - "dataset": dataset.receipt(), + "dataset": {**dataset.receipt(), "selection": selection}, "metrics": { "samples": len(rows), "passed_samples": passed_count, @@ -2249,7 +2253,11 @@ def _vision_language_text_parity( } -def _ocr_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: +def _ocr_samples( + dataset: Dataset, + definition: Mapping[str, Any], + sample_limit: int, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: try: payload = json.loads(dataset.path.read_text(encoding="utf-8")) except json.JSONDecodeError as error: @@ -2258,8 +2266,9 @@ def _ocr_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: if not isinstance(samples, list) or len(samples) < sample_limit: raise QualificationError(f"OCR dataset requires at least {sample_limit} samples") root = dataset.path.parent.resolve() + selected_rows, selection = _select_dataset_rows(samples, definition, sample_limit, "OCR") selected = [] - for index, sample in enumerate(samples[:sample_limit]): + for index, sample in enumerate(selected_rows): if not isinstance(sample, Mapping): raise QualificationError(f"OCR sample {index} must be an object") media = sample.get("media") @@ -2299,7 +2308,7 @@ def _ocr_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: "answers": answers, } ) - return selected + return selected, selection def _ocr_gold_match(text: str, answers: Sequence[str]) -> bool: @@ -2310,12 +2319,13 @@ def _ocr_gold_match(text: str, answers: Sequence[str]) -> bool: def _ocr_text_parity( case: QualificationCase, context: RuntimeContext, + definition: Mapping[str, Any], dataset: Dataset, output: Path, ) -> dict[str, Any]: configured = case.values sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") - selected = _ocr_samples(dataset, sample_limit) + selected, selection = _ocr_samples(dataset, definition, sample_limit) expected = _family_image_reference(case, context, output, selected) request = configured.get("request", {}) if not isinstance(request, Mapping): @@ -2367,7 +2377,7 @@ def _ocr_text_parity( "model": case.model, "benchmark": case.benchmark, "bundle": str(bundle), - "dataset": dataset.receipt(), + "dataset": {**dataset.receipt(), "selection": selection}, "metrics": { "samples": len(rows), "sample_pass_rate": sample_pass_rate, @@ -2770,6 +2780,7 @@ def _image_classification_parity( def _speech_transcription_parity( case: QualificationCase, context: RuntimeContext, + definition: Mapping[str, Any], dataset: Dataset, output: Path, ) -> dict[str, Any]: @@ -2780,9 +2791,12 @@ def _speech_transcription_parity( if not isinstance(requests, list) or len(requests) < sample_limit: raise QualificationError(f"speech dataset requires at least {sample_limit} requests") + selected_requests, selection = _select_dataset_rows( + requests, definition, sample_limit, "speech" + ) selected = [] data_root = context.data_root.resolve() - for index, request in enumerate(requests[:sample_limit]): + for index, request in enumerate(selected_requests): if not isinstance(request, Mapping): raise QualificationError(f"speech request {index} must be an object") audio_value = _speech_audio(request) @@ -2913,7 +2927,7 @@ def _speech_transcription_parity( "model": case.model, "benchmark": case.benchmark, "bundle": str(bundle), - "dataset": dataset.receipt(), + "dataset": {**dataset.receipt(), "selection": selection}, "metrics": { "samples": len(rows), "reference_wer": reference_wer, @@ -3309,6 +3323,41 @@ def _prompt(request: Mapping[str, Any]) -> str: raise QualificationError("text-generation request has neither a prompt nor a user message") +def _select_dataset_rows( + rows: Sequence[Any], + definition: Mapping[str, Any], + count: int, + label: str, +) -> tuple[list[Any], dict[str, Any]]: + selection = definition.get("selection", {}) + if not isinstance(selection, Mapping): + raise QualificationError(f"{label} dataset selection must be an object") + method = str(selection.get("method", "first")) + if method == "first": + if len(rows) < count: + raise QualificationError(f"{label} dataset requires at least {count} samples") + indices = list(range(count)) + elif method == "fixed-indices": + configured = selection.get("indices") + if not isinstance(configured, list) or len(configured) < count: + raise QualificationError( + f"{label} fixed dataset selection requires at least {count} indices" + ) + if any( + isinstance(index, bool) or not isinstance(index, int) or index < 0 + for index in configured + ): + raise QualificationError(f"{label} fixed dataset indices must be nonnegative integers") + if len(set(configured)) != len(configured): + raise QualificationError(f"{label} fixed dataset indices must be unique") + indices = configured[:count] + if any(index >= len(rows) for index in indices): + raise QualificationError(f"{label} fixed dataset index is out of range") + else: + raise QualificationError(f"unsupported {label} dataset selection method {method!r}") + return [rows[index] for index in indices], {"method": method, "indices": indices} + + def _positive_int(value: Any, name: str) -> int: if isinstance(value, bool) or not isinstance(value, int) or value < 1: raise QualificationError(f"{name} must be a positive integer") diff --git a/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml b/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml index f9ce441811..4e8fe3a79e 100644 --- a/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml +++ b/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml @@ -13,7 +13,20 @@ dataset: instructions: https://www.openslr.org/12 license: CC-BY-4.0 selection: - method: first + # Ten evenly distributed utterances from distinct speaker/chapter pairs keep + # the existing runtime without overfitting qualification to one speaker. + method: fixed-indices + indices: + - 131 # 1320-122612-0011 + - 393 # 3575-170457-0024 + - 655 # 8230-279154-0038 + - 917 # 5142-33396-0019 + - 1179 # 4446-2273-0021 + - 1441 # 8224-274384-0004 + - 1703 # 3729-6852-0043 + - 1965 # 1580-141083-0016 + - 2227 # 61-70968-0048 + - 2489 # 5683-32865-0006 input: adapter: audio_path metric: diff --git a/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml b/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml index 31cbfa36ad..8ae0af902d 100644 --- a/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml +++ b/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml @@ -13,7 +13,20 @@ dataset: instructions: https://github.com/hendrycks/test license: MIT selection: - method: first + # Ten subjects keep the existing runtime while covering STEM, medicine, + # humanities, social science, and professional knowledge. + method: fixed-indices + indices: + - 50 # abstract_algebra + - 167 # anatomy + - 437 # business_ethics + - 1046 # college_computer_science + - 1863 # econometrics + - 5970 # high_school_world_history + - 6891 # machine_learning + - 11398 # professional_law + - 13504 # sociology + - 13956 # world_religions input: adapter: mmlu_continuation metric: diff --git a/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml b/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml index 7b9602ce94..32222af0ca 100644 --- a/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml +++ b/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml @@ -13,7 +13,20 @@ dataset: instructions: https://huggingface.co/datasets/ling99/OCRBench_v2 license: MIT selection: - method: first + # Ten task types keep the existing runtime while covering recognition, + # parsing, extraction, reasoning, tables, charts, formulas, and translation. + method: fixed-indices + indices: + - 150 # APP agent en / rico + - 2100 # chart parsing en / OneChart + - 3950 # document parsing cn / CDLA + - 7100 # fine-grained text recognition en / spotting_cocotext + - 4900 # formula recognition en / HME100K + - 5700 # full-page OCR cn / CDLA + - 1100 # key information extraction en / POIE + - 6300 # reasoning VQA en / mmsi + - 8000 # table parsing en / table_to_html + - 9800 # text translation cn / memes input: adapter: ocrbench_v2 metric: diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 746256c030..a1c0c2b7f0 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -323,6 +323,15 @@ def reference(command, *_args, **_kwargs): return subprocess.CompletedProcess(command, 0, stdout="", stderr="") monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: { + "kind": "accuracy", + "selection": {"method": "first"}, + "metric": {"name": "exact_token_ids"}, + }, + ) monkeypatch.setattr( qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) ) @@ -505,9 +514,7 @@ def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> N assert resolved == {"image_path": str(image.resolve()), "batch_size": 1} -def test_robot_action_accuracy_compares_complete_action_chunk( - tmp_path: Path, monkeypatch -) -> None: +def test_robot_action_accuracy_compares_complete_action_chunk(tmp_path: Path, monkeypatch) -> None: data = tmp_path / "families/lerobot_act/tests/data" data.mkdir(parents=True) image = data / "image.png" @@ -516,13 +523,7 @@ def test_robot_action_accuracy_compares_complete_action_chunk( state.write_bytes(b"state") dataset_path = data / "requests.json" dataset_path.write_text( - json.dumps( - { - "requests": [ - {"id": "recorded", "image": image.name, "state": state.name} - ] - } - ), + json.dumps({"requests": [{"id": "recorded", "image": image.name, "state": state.name}]}), encoding="utf-8", ) profile = tmp_path / "families/lerobot_act/tests/benchmark/example.yaml" @@ -965,13 +966,11 @@ def test_localization_accuracy_matches_unordered_boxes_and_points() -> None: ) assert first_kind == second_kind == "box" - assert qualification_accuracy._localization_alignment( - first_boxes, second_boxes, "box" - ) == 1.0 + assert qualification_accuracy._localization_alignment(first_boxes, second_boxes, "box") == 1.0 assert first_point_kind == second_point_kind == "point" - assert qualification_accuracy._localization_alignment( - first_points, second_points, "point" - ) == 10.0 + assert ( + qualification_accuracy._localization_alignment(first_points, second_points, "point") == 10.0 + ) with pytest.raises(QualificationError, match="reference tag"): qualification_accuracy._localization_values("<10><20><100><200>") @@ -1965,6 +1964,53 @@ def test_shared_definitions_own_dataset_and_metric_not_models() -> None: assert definition["dataset"]["id"] +def test_fixed_dataset_selection_uses_declared_indices() -> None: + rows = [f"sample-{index}" for index in range(8)] + + selected, receipt = qualification_accuracy._select_dataset_rows( + rows, + {"selection": {"method": "fixed-indices", "indices": [6, 2, 4]}}, + 2, + "example", + ) + + assert selected == ["sample-6", "sample-2"] + assert receipt == {"method": "fixed-indices", "indices": [6, 2]} + + +@pytest.mark.parametrize( + ("indices", "message"), + [ + ([1, 1], "must be unique"), + ([1, True], "nonnegative integers"), + ([1, 8], "out of range"), + ], +) +def test_fixed_dataset_selection_rejects_invalid_indices(indices: list[int], message: str) -> None: + with pytest.raises(QualificationError, match=message): + qualification_accuracy._select_dataset_rows( + list(range(8)), + {"selection": {"method": "fixed-indices", "indices": indices}}, + 2, + "example", + ) + + +@pytest.mark.parametrize( + "benchmark", + ["mmlu_continuation", "librispeech_transcription", "ocrbench_v2_parity"], +) +def test_slow_dataset_benchmarks_keep_ten_fixed_representative_samples( + benchmark: str, +) -> None: + case = next(case for case in discover(REPOSITORY) if case.benchmark == benchmark) + selection = load_benchmark(REPOSITORY, case)["selection"] + + assert selection["method"] == "fixed-indices" + assert len(selection["indices"]) == 10 + assert len(set(selection["indices"])) == 10 + + def test_shared_performance_definitions_own_complete_reference_timing() -> None: for case in discover(REPOSITORY): if case.kind != "performance": @@ -2370,8 +2416,7 @@ def test_benchmark_launcher_uses_the_family_python(tmp_path: Path, monkeypatch) assert os.access(launcher, os.X_OK) assert launcher.read_text(encoding="utf-8") == ( - "#!/bin/sh\n" - f"exec '{python}' '{bench}' \"$@\"\n" + f"#!/bin/sh\nexec '{python}' '{bench}' \"$@\"\n" ) @@ -2551,9 +2596,7 @@ def test_family_dataset_stays_with_its_discovered_model_profile(tmp_path: Path) resolve_dataset(definition, context, profile) -def test_stereo_accuracy_compares_complete_disparity_artifacts( - tmp_path: Path, monkeypatch -) -> None: +def test_stereo_accuracy_compares_complete_disparity_artifacts(tmp_path: Path, monkeypatch) -> None: profile = tmp_path / "families/stereo/tests/benchmark/stereo.yaml" reference_script = profile.parent / "reference.py" reference_script.parent.mkdir(parents=True) From 94c5e40acbb9922f07aaf088b757211d9b053a45 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 21 Sep 2026 11:29:37 +0000 Subject: [PATCH 092/100] chore(benchmark): split first text batch Move the BLOOM 560M, GPT-Neo 125M, and Pythia 70M qualification profiles to the focused PR #1397. Keep the remaining restoration work in PR #1379 unchanged. Signed-off-by: chaofengw --- .../bloom/tests/benchmark/bloom-560m.yaml | 54 ------------------- .../gpt_neo/tests/benchmark/gpt-neo-125m.yaml | 54 ------------------- .../gpt_neox/tests/benchmark/pythia-70m.yaml | 54 ------------------- 3 files changed, 162 deletions(-) delete mode 100644 families/bloom/tests/benchmark/bloom-560m.yaml delete mode 100644 families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml delete mode 100644 families/gpt_neox/tests/benchmark/pythia-70m.yaml diff --git a/families/bloom/tests/benchmark/bloom-560m.yaml b/families/bloom/tests/benchmark/bloom-560m.yaml deleted file mode 100644 index eb8482c31b..0000000000 --- a/families/bloom/tests/benchmark/bloom-560m.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: bloom-560m -candidate: - family: bloom - checkpoint: bigscience/bloom-560m - task: text_generation - precision: fp32 - bundle: bloom-560m.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-64 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml b/families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml deleted file mode 100644 index 3a9007aee9..0000000000 --- a/families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: gpt-neo-125m -candidate: - family: gpt_neo - checkpoint: EleutherAI/gpt-neo-125m - task: text_generation - precision: fp16 - bundle: gpt-neo-125m.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-64 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/gpt_neox/tests/benchmark/pythia-70m.yaml b/families/gpt_neox/tests/benchmark/pythia-70m.yaml deleted file mode 100644 index a80f8dda2e..0000000000 --- a/families/gpt_neox/tests/benchmark/pythia-70m.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: pythia-70m -candidate: - family: gpt_neox - checkpoint: EleutherAI/pythia-70m - task: text_generation - precision: fp32 - bundle: pythia-70m.bundle - build: - max_sequence_length: 400 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 336 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-64 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids From 4b971a309fbe95753ca22a5ef90c9563e22dd427 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 21 Sep 2026 11:56:32 +0000 Subject: [PATCH 093/100] chore(benchmark): split two text batches Move ten text-model qualification profiles into the focused PRs #1398 and #1399. Keep the remaining restoration work in PR #1379 unchanged. Signed-off-by: chaofengw --- .../falcon/tests/benchmark/falcon-rw-1b.yaml | 55 ---------- .../tests/benchmark/granite-3.1-2b.yaml | 55 ---------- .../llama/tests/benchmark/tinyllama-1.1b.yaml | 56 ---------- .../mamba/tests/benchmark/mamba-130m.yaml | 54 ---------- .../tests/benchmark/mixtral-stories-15m.yaml | 55 ---------- families/olmo/tests/benchmark/olmo-1b.yaml | 55 ---------- families/olmo2/tests/benchmark/olmo2-1b.yaml | 99 ----------------- families/opt/tests/benchmark/opt-125m.yaml | 55 ---------- families/rwkv/tests/benchmark/rwkv-169m.yaml | 55 ---------- .../tests/benchmark/stablelm2-1.6b.yaml | 100 ------------------ 10 files changed, 639 deletions(-) delete mode 100644 families/falcon/tests/benchmark/falcon-rw-1b.yaml delete mode 100644 families/granite/tests/benchmark/granite-3.1-2b.yaml delete mode 100644 families/llama/tests/benchmark/tinyllama-1.1b.yaml delete mode 100644 families/mamba/tests/benchmark/mamba-130m.yaml delete mode 100644 families/mixtral/tests/benchmark/mixtral-stories-15m.yaml delete mode 100644 families/olmo/tests/benchmark/olmo-1b.yaml delete mode 100644 families/olmo2/tests/benchmark/olmo2-1b.yaml delete mode 100644 families/opt/tests/benchmark/opt-125m.yaml delete mode 100644 families/rwkv/tests/benchmark/rwkv-169m.yaml delete mode 100644 families/stablelm/tests/benchmark/stablelm2-1.6b.yaml diff --git a/families/falcon/tests/benchmark/falcon-rw-1b.yaml b/families/falcon/tests/benchmark/falcon-rw-1b.yaml deleted file mode 100644 index a16e55b83a..0000000000 --- a/families/falcon/tests/benchmark/falcon-rw-1b.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: falcon-rw-1b -candidate: - family: falcon - checkpoint: tiiuae/falcon-rw-1b - task: text_generation - precision: fp32 - bundle: falcon-rw-1b.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-20 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The quick brown fox jumps over the lazy dog. Once upon a time - max_new_tokens: 20 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/granite/tests/benchmark/granite-3.1-2b.yaml b/families/granite/tests/benchmark/granite-3.1-2b.yaml deleted file mode 100644 index 15e351fe3d..0000000000 --- a/families/granite/tests/benchmark/granite-3.1-2b.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: granite-3.1-2b -candidate: - family: granite - checkpoint: ibm-granite/granite-3.1-2b-base - task: text_generation - precision: fp16 - bundle: granite-3.1-2b.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-20 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - max_new_tokens: 20 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/llama/tests/benchmark/tinyllama-1.1b.yaml b/families/llama/tests/benchmark/tinyllama-1.1b.yaml deleted file mode 100644 index b29c87ce16..0000000000 --- a/families/llama/tests/benchmark/tinyllama-1.1b.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: tinyllama-1.1b -candidate: - family: llama - checkpoint: TinyLlama/TinyLlama-1.1B-Chat-v1.0 - task: text_generation - precision: fp16 - bundle: tinyllama-1.1b.bundle - build: - max_sequence_length: 258 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-10 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: What is the capital of France? Answer in one word. - max_new_tokens: 10 - use_chat_template: true - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/mamba/tests/benchmark/mamba-130m.yaml b/families/mamba/tests/benchmark/mamba-130m.yaml deleted file mode 100644 index ebe75ab705..0000000000 --- a/families/mamba/tests/benchmark/mamba-130m.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: mamba-130m -candidate: - family: mamba - checkpoint: state-spaces/mamba-130m-hf - task: text_generation - precision: fp32 - bundle: mamba-130m.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-64 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/mixtral/tests/benchmark/mixtral-stories-15m.yaml b/families/mixtral/tests/benchmark/mixtral-stories-15m.yaml deleted file mode 100644 index 83b4cb1376..0000000000 --- a/families/mixtral/tests/benchmark/mixtral-stories-15m.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: mixtral-stories-15m -candidate: - family: mixtral - checkpoint: ggml-org/stories15M_MOE - task: text_generation - precision: fp16 - bundle: mixtral-stories-15m.bundle - build: - max_sequence_length: 256 - fp32_layers: [3, 4, 5] - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-64 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: Once upon a time there was a little - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/olmo/tests/benchmark/olmo-1b.yaml b/families/olmo/tests/benchmark/olmo-1b.yaml deleted file mode 100644 index d33e4f7315..0000000000 --- a/families/olmo/tests/benchmark/olmo-1b.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: olmo-1b -candidate: - family: olmo - checkpoint: allenai/OLMo-1B-hf - task: text_generation - precision: fp16 - bundle: olmo-1b.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-20 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - max_new_tokens: 20 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/olmo2/tests/benchmark/olmo2-1b.yaml b/families/olmo2/tests/benchmark/olmo2-1b.yaml deleted file mode 100644 index 8e7f23da7c..0000000000 --- a/families/olmo2/tests/benchmark/olmo2-1b.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: olmo2-1b -candidate: - family: olmo2 - checkpoint: allenai/OLMo-2-0425-1B - revision: a1847dff35000b4271fa70afc5db10fd29fedbdf - task: text_generation - precision: fp32 - bundle: olmo2-1b.bundle - build: - max_sequence_length: 352 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-8 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: |- - The following are multiple choice questions (with answers) about abstract algebra. - - Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field. - A. 0 - B. 1 - C. 2 - D. 3 - Answer: B - - Statement 1 | If aH is an element of a factor group, then |aH| divides |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: B - - Statement 1 | Every element of a group generates a cyclic subgroup of the group. Statement 2 | The symmetric group S_10 has 10 elements. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: C - - Statement 1| Every function from a finite set onto itself must be one to one. Statement 2 | Every subgroup of an abelian group is abelian. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: A - - Find the characteristic of the ring 2Z. - A. 0 - B. 3 - C. 12 - D. 30 - Answer: A - - Find the degree for the given field extension Q(sqrt(2), sqrt(3), sqrt(18)) over Q. - A. 0 - B. 4 - C. 2 - D. 6 - Answer: - max_new_tokens: 8 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/opt/tests/benchmark/opt-125m.yaml b/families/opt/tests/benchmark/opt-125m.yaml deleted file mode 100644 index fa343f9ec3..0000000000 --- a/families/opt/tests/benchmark/opt-125m.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: opt-125m -candidate: - family: opt - checkpoint: facebook/opt-125m - task: text_generation - precision: fp16 - bundle: opt-125m.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-10 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - max_new_tokens: 10 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/rwkv/tests/benchmark/rwkv-169m.yaml b/families/rwkv/tests/benchmark/rwkv-169m.yaml deleted file mode 100644 index 39942ec515..0000000000 --- a/families/rwkv/tests/benchmark/rwkv-169m.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: rwkv-169m -candidate: - family: rwkv - checkpoint: RWKV/rwkv-4-169m-pile - task: text_generation - precision: fp32 - bundle: rwkv-169m.bundle - build: - max_sequence_length: 256 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-10 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - max_new_tokens: 10 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/stablelm/tests/benchmark/stablelm2-1.6b.yaml b/families/stablelm/tests/benchmark/stablelm2-1.6b.yaml deleted file mode 100644 index 54149dfe34..0000000000 --- a/families/stablelm/tests/benchmark/stablelm2-1.6b.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: stablelm2-1.6b -candidate: - family: stablelm - checkpoint: stabilityai/stablelm-2-1_6b - revision: f499ead74c53749bd93cebc6ce8bc0d7bdf1eaef - task: text_generation - precision: fp16 - bundle: stablelm2-1.6b.bundle - build: - max_sequence_length: 384 - fp32_layers: [23] - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp16 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-22 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: |- - The following are multiple choice questions (with answers) about abstract algebra. - - Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field. - A. 0 - B. 1 - C. 2 - D. 3 - Answer: B - - Statement 1 | If aH is an element of a factor group, then |aH| divides |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: B - - Statement 1 | Every element of a group generates a cyclic subgroup of the group. Statement 2 | The symmetric group S_10 has 10 elements. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: C - - Statement 1| Every function from a finite set onto itself must be one to one. Statement 2 | Every subgroup of an abelian group is abelian. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: A - - Find the characteristic of the ring 2Z. - A. 0 - B. 3 - C. 12 - D. 30 - Answer: A - - Let p = (1, 2, 5, 4)(2, 3) in S_5 . Find the index of

in S_5. - A. 8 - B. 2 - C. 24 - D. 120 - Answer: - max_new_tokens: 22 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp16 - output_contract: exact-token-ids From 0136f3b8f11e9a66a9da53da8fc26320f321d9b8 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 06:49:01 +0000 Subject: [PATCH 094/100] fix(benchmark): decouple family qualification policy Resolve reference inputs from suite and family declarations instead of shared registries. Reject duplicate qualification YAML keys and keep family-specific regression checks in their owning test slices. Signed-off-by: chaofengw --- apps/benchmark/performance/README.md | 20 ++ .../performance/baselines/task_reference.py | 13 +- apps/benchmark/performance/release.yaml | 69 +++--- .../trtmc_benchmark/tests/test_perf_matrix.py | 16 +- .../tests/benchmark/lance-3b-x2t-image.yaml | 2 +- .../lance/tests/test_official_reference.py | 9 + .../tests/test_translation_contract.py | 14 ++ .../nemotron_h/tests/test_runtime_contract.py | 9 + .../tests/benchmark/personaplex-7b.yaml | 2 +- .../tests/test_official_reference.py | 18 ++ .../benchmark/sana-wm-bidirectional.yaml | 2 +- .../sana_wm/tests/test_reference_contract.py | 2 +- tools/benchmark_qualification/accuracy.py | 15 +- tools/benchmark_qualification/catalog.py | 31 ++- tools/benchmark_qualification/performance.py | 22 +- tools/benchmark_qualification/runtime.py | 16 ++ tools/perf_matrix.py | 100 +++++---- tools/tests/test_model_benchmark.py | 212 +++++++++++------- 18 files changed, 365 insertions(+), 207 deletions(-) diff --git a/apps/benchmark/performance/README.md b/apps/benchmark/performance/README.md index 754f30c28d..458ab45ebd 100644 --- a/apps/benchmark/performance/README.md +++ b/apps/benchmark/performance/README.md @@ -119,6 +119,26 @@ central exclusions for validation; it does not establish qualification. Use an existing reference adapter when it implements the actual workload. Otherwise declare a Python script relative to the owning family directory: +Reference adapters that need prepared repositories or model directories declare +their inputs on the suite entry instead of relying on a shared adapter registry. +Each option names a field from the selected performance environment; optional +`required` paths are checked below that directory before the reference starts: + +```yaml +baseline: + runner: task-reference + adapter: upstream-example + reference_inputs: + reference_repo: + environment: example_repo + required: [inference.py] +``` + +The environment supplies `references.example_repo`. A family qualification +profile may instead put `reference_repo` directly under +`reference_environment.paths`; the qualification runner forwards all such +family-declared paths as adapter options without knowing their meaning. + ```yaml schema_version: trtmc.perf-suite/v2 name: example-performance diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index a4ba428788..a08d223b21 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -1707,18 +1707,7 @@ def _load_vision( kwargs = _load_kwargs(arguments, torch) processor_kwargs = _processor_kwargs(arguments) - if arguments.adapter == "timm-classification" or arguments.family in { - "timm_densenet", - "timm_efficientnet", - "timm_inception", - "timm_mnasnet", - "timm_mobilenetv2", - "timm_mobilenetv3", - "timm_repvgg", - "timm_resnet", - "timm_vgg", - "timm_vit", - }: + if arguments.adapter == "timm-classification": import timm from timm.data import create_transform, resolve_model_data_config diff --git a/apps/benchmark/performance/release.yaml b/apps/benchmark/performance/release.yaml index 5e15dea3a7..32ffca3d88 100644 --- a/apps/benchmark/performance/release.yaml +++ b/apps/benchmark/performance/release.yaml @@ -412,6 +412,10 @@ entries: baseline: runner: task-reference adapter: upstream-fast-foundation-stereo + reference_inputs: + model_dir: + environment: fast_foundation_stereo_model + required: [core/foundation_stereo.py, core/submodule.py, weights/23-36-37/model_best_bp2_serialize.pth] mode: pytorch-eager reference_backend: official_pytorch timing_scope: task-pipeline-call-wall @@ -551,6 +555,8 @@ entries: baseline: runner: task-reference adapter: upstream-lance + reference_inputs: + reference_repo: {environment: lance_repo, required: [inference_lance.py]} mode: pytorch-eager reference_backend: upstream_lance timing_scope: task-pipeline-call-wall @@ -569,6 +575,8 @@ entries: baseline: runner: task-reference adapter: pytorch-lerobot-act + reference_inputs: + source_root: {environment: lerobot_repo, required: [lerobot/common/policies/act/modeling_act.py]} mode: pytorch-eager precision: fp32 reference_backend: official_lerobot @@ -805,6 +813,8 @@ entries: baseline: runner: task-reference adapter: pytorch-personaplex + reference_inputs: + official_repo: {environment: personaplex_repo, required: [moshi]} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-pipeline-call-wall @@ -1004,6 +1014,9 @@ entries: baseline: runner: task-reference adapter: upstream-sana-wm + reference_inputs: + reference_repo: {environment: sana_repo} + model_dir: {environment: sana_model} mode: pytorch-eager reference_backend: official_sana timing_scope: task-pipeline-call-wall @@ -1076,7 +1089,7 @@ entries: testcase: timm-vit-base-p16-224-augreg-in21k-ft-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1089,7 +1102,7 @@ entries: testcase: resnet50-a1-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1102,7 +1115,7 @@ entries: testcase: xception41-tf-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1115,7 +1128,7 @@ entries: testcase: regnety-040-ra3-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1128,7 +1141,7 @@ entries: testcase: seresnet50-a1-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1141,7 +1154,7 @@ entries: testcase: senet154-gluon-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1154,7 +1167,7 @@ entries: testcase: ghostnet-100-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1167,7 +1180,7 @@ entries: testcase: convnext-tiny-in12k-ft-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1180,7 +1193,7 @@ entries: testcase: inception-v4-tf-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1193,7 +1206,7 @@ entries: testcase: inception-resnet-v2-tf-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1206,7 +1219,7 @@ entries: testcase: hrnet-w18-ms-aug-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1219,7 +1232,7 @@ entries: testcase: resnest50d-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1232,7 +1245,7 @@ entries: testcase: swin-tiny-patch4-window7-224-ms-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1245,7 +1258,7 @@ entries: testcase: crossvit-9-240-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1258,7 +1271,7 @@ entries: testcase: dpn68b-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1271,7 +1284,7 @@ entries: testcase: res2net50-26w-4s-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1297,7 +1310,7 @@ entries: testcase: dm-nfnet-f0-dm-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1310,7 +1323,7 @@ entries: testcase: mobilevit-s-cvnets-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1349,7 +1362,7 @@ entries: testcase: xcit-tiny-12-p16-224-fb-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1362,7 +1375,7 @@ entries: testcase: repvgg-a2-rvgg-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1375,7 +1388,7 @@ entries: testcase: inception-v3-tv-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1388,7 +1401,7 @@ entries: testcase: mnasnet-100-rmsp-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1401,7 +1414,7 @@ entries: testcase: mobilenetv2-100-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1414,7 +1427,7 @@ entries: testcase: densenet121-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1427,7 +1440,7 @@ entries: testcase: efficientnet-b0-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1440,7 +1453,7 @@ entries: testcase: mobilenetv4-conv-small-050 baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1453,7 +1466,7 @@ entries: testcase: mobilenetv3-large-100-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1466,7 +1479,7 @@ entries: testcase: vgg16-tv-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 16dcaa40cb..b18ceaa7b8 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -1245,8 +1245,14 @@ def _environment(tmp_path: Path) -> tuple[Path, perf.Environment]: "libtrtmc_model_lance.so", ): (runtime / name).write_bytes(b"") + _, suite_entries, _ = perf._load_suite_file(SUITE) + reference_fields = { + str(declaration["environment"]) + for entry in suite_entries + for declaration in entry.get("baseline", {}).get("reference_inputs", {}).values() + } references = {} - for name in perf.REFERENCE_FIELDS: + for name in reference_fields: path = tmp_path / name path.mkdir() references[name] = str(path) @@ -1463,13 +1469,15 @@ def test_release_suite_expands_profiles_and_covers_ready_catalog() -> None: vision_entries = {entry["id"]: entry for entry in entries if entry["id"] in vision_ids} assert set(vision_entries) == vision_ids perf._coverage(entries, excluded) - # Adapter identity belongs to the builtin fixture, not an owned replacement. + # Adapter identity belongs to the suite entry, not a shared family-name registry. _, builtin_entries, _ = perf._load_suite_file(SUITE) builtin_vision = {entry["id"]: entry for entry in builtin_entries if entry["id"] in vision_ids} assert set(builtin_vision) == vision_ids + builtin_timm = [entry for entry in builtin_entries if entry["family"].startswith("timm_")] + assert builtin_timm assert all( - entry["baseline"]["adapter"] == "hf-transformers-vision" - for entry in builtin_vision.values() + entry["baseline"]["adapter"] == "timm-classification" + for entry in builtin_timm ) diff --git a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml index b891f7c119..bcb562c623 100644 --- a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml +++ b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml @@ -6,7 +6,7 @@ candidate: {family: lance, checkpoint: bytedance-research/Lance, revision: 73953 reference_environment: requirements: ../../requirements.txt build_isolation: false - paths: {lance_repo: trtmc-reference/Lance} + paths: {reference_repo: trtmc-reference/Lance} accuracy: - name: generated-text-parity benchmark: task_output_parity diff --git a/families/lance/tests/test_official_reference.py b/families/lance/tests/test_official_reference.py index 05a564d2dc..30bd39b2cd 100644 --- a/families/lance/tests/test_official_reference.py +++ b/families/lance/tests/test_official_reference.py @@ -8,6 +8,7 @@ from types import SimpleNamespace import pytest +import yaml from families.lance.tests.official_reference import ( _image_only_source, @@ -18,6 +19,14 @@ from families.lance.tests.benchmark import prepare_environment +def test_qualification_environment_keeps_extension_builds_in_family_environment() -> None: + profile = yaml.safe_load( + (Path(__file__).parent / "benchmark/lance-3b-x2t-image.yaml").read_text(encoding="utf-8") + ) + + assert profile["reference_environment"]["build_isolation"] is False + + def test_reference_source_is_required_and_fail_closed(monkeypatch, tmp_path) -> None: monkeypatch.delenv("TRTMC_REFERENCE_SOURCE_DIR", raising=False) with pytest.raises(AssertionError, match="TRTMC_REFERENCE_SOURCE_DIR"): diff --git a/families/m2m_100/tests/test_translation_contract.py b/families/m2m_100/tests/test_translation_contract.py index 467bb42183..82e75af64d 100644 --- a/families/m2m_100/tests/test_translation_contract.py +++ b/families/m2m_100/tests/test_translation_contract.py @@ -4,10 +4,24 @@ import json from pathlib import Path +import yaml + TEST_ROOT = Path(__file__).resolve().parent +def test_qualification_profile_owns_source_language_placement() -> None: + profile = yaml.safe_load( + (TEST_ROOT / "benchmark/nllb-200-distilled-600m.yaml").read_text(encoding="utf-8") + ) + + cases = [*profile["accuracy"], *profile["performance"]] + assert cases + assert all( + case["reference"]["source_language_placement"] == "replace-final-unk" for case in cases + ) + + def test_nllb_case_keeps_the_explicit_english_to_french_contract() -> None: manifest = json.loads((TEST_ROOT / "manifests/nllb-200.json").read_text(encoding="utf-8")) case = manifest["testcases"][0] diff --git a/families/nemotron_h/tests/test_runtime_contract.py b/families/nemotron_h/tests/test_runtime_contract.py index e286648ca2..25a6d58e56 100644 --- a/families/nemotron_h/tests/test_runtime_contract.py +++ b/families/nemotron_h/tests/test_runtime_contract.py @@ -8,10 +8,19 @@ from types import SimpleNamespace import pytest +import yaml from families.nemotron_h.model import _runtime_config, _stop_token_ids +def test_qualification_environment_keeps_extension_builds_in_family_environment() -> None: + profile = yaml.safe_load( + (Path(__file__).parent / "benchmark/nemotron-h-nano-9b.yaml").read_text(encoding="utf-8") + ) + + assert profile["reference_environment"]["build_isolation"] is False + + def test_runtime_keeps_a_scalar_eos_and_every_declared_stop_token(tmp_path) -> None: (tmp_path / "generation_config.json").write_text( json.dumps({"eos_token_id": [2, 11, 12]}), encoding="utf-8" diff --git a/families/personaplex/tests/benchmark/personaplex-7b.yaml b/families/personaplex/tests/benchmark/personaplex-7b.yaml index ae07c3a6ff..ddde6f1a00 100644 --- a/families/personaplex/tests/benchmark/personaplex-7b.yaml +++ b/families/personaplex/tests/benchmark/personaplex-7b.yaml @@ -5,7 +5,7 @@ model: personaplex-7b candidate: {family: personaplex, checkpoint: nvidia/personaplex-7b-v1, revision: fdaf4090a61cb315c138a1faee287ffd6c716309, task: speech_to_speech, precision: fp16, bundle: personaplex-7b.bundle, build: {max_sequence_length: 512, fp32_layers: [0, 1]}} reference_environment: requirements: ../../requirements.txt - paths: {personaplex_repo: trtmc-reference/personaplex} + paths: {official_repo: trtmc-reference/personaplex} accuracy: - name: generated-audio-parity benchmark: task_output_parity diff --git a/families/personaplex/tests/test_official_reference.py b/families/personaplex/tests/test_official_reference.py index b14c2fcf36..d29af04f46 100644 --- a/families/personaplex/tests/test_official_reference.py +++ b/families/personaplex/tests/test_official_reference.py @@ -14,6 +14,7 @@ import pytest from . import official_reference +from .benchmark import prepare_environment from .personaplex_audio_compat import sphn @@ -21,6 +22,23 @@ SOURCE_REVISION = "3428dfd95309a7f3c84fd93259ded0f810d1ff91" +def test_qualification_environment_installs_official_audio_dependency(monkeypatch) -> None: + calls = [] + monkeypatch.setattr(prepare_environment, "package_version", lambda _name: "0.1.3") + monkeypatch.setattr( + prepare_environment.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)), + ) + + prepare_environment._install_sphn() + + command, kwargs = calls[0] + assert command[-1] == "sphn==0.1.4" + assert kwargs["check"] is True + assert kwargs["env"]["CMAKE_POLICY_VERSION_MINIMUM"] == "3.5" + + def _write_wav(path: Path) -> None: with wave.open(str(path), "wb") as output: output.setnchannels(1) diff --git a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml index 423c468a07..a061e206a2 100644 --- a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml +++ b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml @@ -5,7 +5,7 @@ model: sana-wm-bidirectional candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, revision: e96271d77398def8ebb9fc595e7c0056dc625ab7, model_directory: trtmc-reference/SANA-model, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} reference_environment: requirements: ../../requirements.txt - paths: {sana_repo: trtmc-reference/Sana, sana_model: trtmc-reference/SANA-model} + paths: {reference_repo: trtmc-reference/Sana, model_dir: trtmc-reference/SANA-model} accuracy: - name: generated-media-parity benchmark: task_output_parity diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index 2eb0c7e513..5b0ea99d55 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -22,7 +22,7 @@ def test_qualification_candidate_uses_the_prepared_family_model() -> None: prepared_model = "trtmc-reference/SANA-model" assert profile["candidate"]["model_directory"] == prepared_model - assert profile["reference_environment"]["paths"]["sana_model"] == prepared_model + assert profile["reference_environment"]["paths"]["model_dir"] == prepared_model def test_native_plugin_uses_installed_tensorrt_library(tmp_path: Path) -> None: diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 90edcddb4f..96bc3a38a7 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -34,8 +34,8 @@ RuntimeContext, benchmark_executable, prepare_bundle, + reference_environment_options, reference_python, - reference_environment_paths, require_candidate, run_command, write_model_descriptor, @@ -95,13 +95,6 @@ def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, return result -_REFERENCE_OPTIONS = { - "upstream-lance": {"lance_repo": "reference_repo"}, - "upstream-sana-wm": {"sana_repo": "reference_repo", "sana_model": "model_dir"}, - "pytorch-personaplex": {"personaplex_repo": "official_repo"}, -} - - def _task_output_parity( case: QualificationCase, context: RuntimeContext, output: Path ) -> dict[str, Any]: @@ -128,11 +121,7 @@ def _task_output_parity( if not path.is_relative_to(family_root) or not path.is_file(): raise QualificationError(f"task-output reference asset {name!r} is unavailable: {path}") adapter_options[name] = str(path) - environment_paths = reference_environment_paths(case, context) - for source, target in _REFERENCE_OPTIONS.get(adapter, {}).items(): - if source not in environment_paths: - raise QualificationError(f"task-output reference path {source!r} is not configured") - adapter_options[target] = environment_paths[source] + adapter_options = reference_environment_options(case, context, adapter_options) reference_path = output / "reference-result.json" command = [ str(reference_python(case, context)), diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index 90ce84ee5c..c9546eae76 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -19,6 +19,35 @@ class QualificationError(RuntimeError): pass +class _UniqueKeyLoader(yaml.SafeLoader): + """Load policy YAML while rejecting duplicate explicit mapping keys.""" + + def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Any, Any]: + seen: set[Any] = set() + for key_node, _ in node.value: + if key_node.tag == "tag:yaml.org,2002:merge": + continue + key = self.construct_object(key_node, deep=False) + try: + duplicate = key in seen + seen.add(key) + except TypeError as error: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable mapping key", + key_node.start_mark, + ) from error + if duplicate: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + return super().construct_mapping(node, deep=deep) + + @dataclass(frozen=True) class QualificationCase: kind: str @@ -214,7 +243,7 @@ def _reference_paths(raw: Any, path: Path) -> dict[str, str]: def _yaml_object(path: Path, label: str) -> dict[str, Any]: try: - value = yaml.safe_load(path.read_text(encoding="utf-8")) + value = yaml.load(path.read_text(encoding="utf-8"), Loader=_UniqueKeyLoader) except (OSError, yaml.YAMLError) as error: raise QualificationError(f"cannot read {label} {path}: {error}") from error if not isinstance(value, dict): diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py index bd36ae680f..80e4ebac9d 100644 --- a/tools/benchmark_qualification/performance.py +++ b/tools/benchmark_qualification/performance.py @@ -17,7 +17,7 @@ from .runtime import ( RuntimeContext, benchmark_executable, - reference_environment_paths, + reference_environment_options, reference_python, require_candidate, run_command, @@ -88,7 +88,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st "Performance measurement.iterations must satisfy the stability sample count" ) request = _resolve_family_assets(case, request) - baseline = _resolve_reference_assets(case, baseline) + baseline = _resolve_reference_assets(case, context, baseline) descriptor = write_model_descriptor(case, output, request, context=context) entry_id = f"qualification.{case.family}.{case.name}" suite = { @@ -127,16 +127,6 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st suite_path = output / "resolved-suite.yaml" suite_path.write_text(yaml.safe_dump(suite, sort_keys=False), encoding="utf-8") results_root = output / "matrix" - references = { - "elf_repo": "", - "lance_repo": "", - "lerobot_repo": "", - "sana_repo": "", - "sana_model": "", - "personaplex_repo": "", - "fast_foundation_stereo_model": "", - } - references.update(reference_environment_paths(case, context)) environment = { "schema_version": "trtmc.perf-environment/v2", "name": "qualification", @@ -151,7 +141,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st ), "reference_python": str(reference_python(case, context)), }, - "references": references, + "references": {}, "storage": { "results_root": str(results_root), "scratch_root": str(output / "scratch"), @@ -267,7 +257,9 @@ def _resolve_family_assets(case: QualificationCase, request: Mapping[str, Any]) def _resolve_reference_assets( - case: QualificationCase, reference: Mapping[str, Any] + case: QualificationCase, + context: RuntimeContext, + reference: Mapping[str, Any], ) -> dict[str, Any]: resolved = dict(reference) options = reference.get("adapter_options", {}) @@ -284,5 +276,5 @@ def _resolve_reference_assets( if not path.is_relative_to(family_root) or not path.is_file(): raise QualificationError(f"Performance reference asset {key!r} is unavailable: {path}") resolved_options[key] = str(path) - resolved["adapter_options"] = resolved_options + resolved["adapter_options"] = reference_environment_options(case, context, resolved_options) return resolved diff --git a/tools/benchmark_qualification/runtime.py b/tools/benchmark_qualification/runtime.py index 63b020f1e2..ce3bf2f903 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/tools/benchmark_qualification/runtime.py @@ -286,6 +286,22 @@ def reference_environment_paths(case: QualificationCase, context: RuntimeContext return result +def reference_environment_options( + case: QualificationCase, + context: RuntimeContext, + configured: Mapping[str, Any], +) -> dict[str, Any]: + """Add family-declared reference paths without knowing their meaning.""" + result = dict(configured) + paths = reference_environment_paths(case, context) + if duplicates := sorted(result.keys() & paths.keys()): + raise QualificationError( + "reference options duplicate prepared environment paths: " + ", ".join(duplicates) + ) + result.update(paths) + return result + + def benchmark_executable(case: QualificationCase, context: RuntimeContext) -> Path: """Run user-facing benchmark and bundle build with the family Python environment.""" python = reference_python(case, context) diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index d4ca6a0938..89707c3771 100755 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -101,28 +101,6 @@ "transcription-text", "vision-language-text", } -REFERENCE_INPUTS = { - "pytorch-lerobot-act": (("source_root", "lerobot_repo"),), - "upstream-elf": (("reference_repo", "elf_repo"),), - "upstream-lance": (("reference_repo", "lance_repo"),), - "upstream-sana-wm": ( - ("reference_repo", "sana_repo"), - ("model_dir", "sana_model"), - ), - "pytorch-personaplex": (("official_repo", "personaplex_repo"),), - "upstream-fast-foundation-stereo": (("model_dir", "fast_foundation_stereo_model"),), -} -REFERENCE_FIELDS = { - "elf_repo", - "lance_repo", - "lerobot_repo", - "sana_repo", - "sana_model", - "personaplex_repo", - "fast_foundation_stereo_model", -} - - class PerfMatrixError(RuntimeError): pass @@ -383,6 +361,35 @@ def _validate_entry(entry: Mapping[str, Any]) -> None: _family_script(entry) elif baseline["runner"] == "task-reference" and not isinstance(baseline.get("adapter"), str): raise PerfMatrixError(f"entry {entry['id']} requires baseline.adapter") + reference_inputs = baseline.get("reference_inputs", {}) + if not isinstance(reference_inputs, Mapping): + raise PerfMatrixError(f"entry {entry['id']} baseline.reference_inputs must be an object") + for option_name, declaration in reference_inputs.items(): + if not isinstance(option_name, str) or not option_name: + raise PerfMatrixError( + f"entry {entry['id']} baseline.reference_inputs keys must be non-empty strings" + ) + if not isinstance(declaration, Mapping): + raise PerfMatrixError( + f"entry {entry['id']} reference input {option_name!r} must be an object" + ) + environment_field = declaration.get("environment") + required = declaration.get("required", []) + if not isinstance(environment_field, str) or not environment_field: + raise PerfMatrixError( + f"entry {entry['id']} reference input {option_name!r} requires environment" + ) + if not isinstance(required, list) or not all( + isinstance(path, str) + and path + and not Path(path).is_absolute() + and ".." not in Path(path).parts + for path in required + ): + raise PerfMatrixError( + f"entry {entry['id']} reference input {option_name!r} required files " + "must be relative paths" + ) fallback = baseline.get("fallback") if fallback is not None and (not isinstance(fallback, str) or not fallback): raise PerfMatrixError(f"entry {entry['id']} baseline.fallback must be a mode") @@ -474,10 +481,11 @@ def load_environment(path: Path) -> Environment: raise PerfMatrixError("environment tools is missing: " + ", ".join(missing)) if missing := sorted(required_storage - storage.keys()): raise PerfMatrixError("environment storage is missing: " + ", ".join(missing)) - if missing := sorted(REFERENCE_FIELDS - references.keys()): - raise PerfMatrixError("environment references is missing: " + ", ".join(missing)) - if not all(isinstance(references[name], str) for name in REFERENCE_FIELDS): - raise PerfMatrixError("environment reference inputs must be strings") + if not all( + isinstance(name, str) and name and isinstance(value, str) + for name, value in references.items() + ): + raise PerfMatrixError("environment reference inputs must map names to strings") timeout = execution.get("timeout_seconds", 7200) if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout < 1: raise PerfMatrixError("execution.timeout_seconds must be positive") @@ -515,7 +523,7 @@ def load_environment(path: Path) -> Environment: bundle_retention=retention, local_files_only=bool(execution.get("local_files_only", False)), timeout_seconds=timeout, - references={name: str(references[name]) for name in REFERENCE_FIELDS}, + references={str(name): str(value) for name, value in references.items()}, storage_root=( _path(storage_root, "storage.storage_root") if storage_root is not None else None ), @@ -812,37 +820,31 @@ def _adapter_options(entry: ResolvedEntry, environment: Environment) -> dict[str if not isinstance(configured, Mapping): raise PerfMatrixError(f"entry {entry.spec['id']} adapter_options must be an object") options = dict(configured) - inputs = REFERENCE_INPUTS.get(str(entry.spec["baseline"].get("adapter", "")), ()) - for option_name, field in inputs: + inputs = entry.spec["baseline"].get("reference_inputs", {}) + for option_name, declaration in inputs.items(): + field = str(declaration["environment"]) + if field not in environment.references: + raise PerfMatrixError( + f"entry {entry.spec['id']} environment references is missing: {field}" + ) path = _path(environment.references[field], f"references.{field}") if not path.is_dir(): raise PerfMatrixError( f"entry {entry.spec['id']} reference path is not a directory: {path}" ) - _validate_reference_path(entry, field, path) + missing = [ + relative + for relative in declaration.get("required", []) + if not (path / relative).exists() + ] + if missing: + raise PerfMatrixError( + f"entry {entry.spec['id']} reference path {path} is missing: " + ", ".join(missing) + ) options[option_name] = str(path) return options -def _validate_reference_path(entry: ResolvedEntry, field: str, path: Path) -> None: - required = { - "elf_repo": ("src",), - "lance_repo": ("inference_lance.py",), - "lerobot_repo": ("lerobot/common/policies/act/modeling_act.py",), - "personaplex_repo": ("moshi",), - "fast_foundation_stereo_model": ( - "core/foundation_stereo.py", - "core/submodule.py", - "weights/23-36-37/model_best_bp2_serialize.pth", - ), - }.get(field, ()) - missing = [relative for relative in required if not (path / relative).exists()] - if missing: - raise PerfMatrixError( - f"entry {entry.spec['id']} reference path {path} is missing: " + ", ".join(missing) - ) - - def _baseline_mode(baseline: Mapping[str, Any]) -> str: return str(baseline.get("mode", "reference" if "script" in baseline else "torch-compile")) diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index a1c0c2b7f0..2feac9b14a 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -17,9 +17,9 @@ from tools import model_benchmark, prepare_coco_detection_dataset from apps.benchmark.performance.baselines import hf_transformers, task_reference from apps.benchmark.performance.baselines.timing_contracts import timing_contract -from families.personaplex.tests.benchmark import prepare_environment as personaplex_environment from tools.benchmark_qualification import accuracy as qualification_accuracy from tools.benchmark_qualification import performance as qualification_performance +from tools.benchmark_qualification import runtime as qualification_runtime from tools.benchmark_qualification.catalog import ( QualificationCase, QualificationError, @@ -32,6 +32,7 @@ RuntimeContext, benchmark_executable, prepare_bundle, + reference_environment_options, reference_python, run_command, write_model_descriptor, @@ -41,6 +42,86 @@ REPOSITORY = Path(__file__).resolve().parents[2] +def _example_case(tmp_path: Path, *, kind: str = "accuracy") -> QualificationCase: + return QualificationCase( + kind=kind, + model="example-model", + family="example", + name="parity", + benchmark="example_benchmark", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "example_task", + "precision": "fp32", + "build": {}, + }, + values={}, + source=tmp_path / "families/example/tests/benchmark/example-model.yaml", + reference_requirements=None, + ) + + +def test_qualification_yaml_rejects_duplicate_explicit_keys(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-model.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\nmodel: example-model\nmodel: replaced-model\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="duplicate key 'model'"): + discover(tmp_path) + + +def test_qualification_yaml_preserves_merge_keys(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-model.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example-model\n" + "candidate: {family: example, checkpoint: example/model, task: example_task, " + "precision: fp32}\n" + "reference: &reference {runner: hf-transformers, mode: hf-eager}\n" + "accuracy: []\n" + "performance:\n" + " - name: parity\n" + " benchmark: example_performance\n" + " reference: {<<: *reference, precision: fp32}\n", + encoding="utf-8", + ) + + cases = discover(tmp_path) + + assert len(cases) == 1 + assert cases[0].values["reference"] == { + "runner": "hf-transformers", + "mode": "hf-eager", + "precision": "fp32", + } + + +def test_reference_environment_options_are_family_declared(tmp_path: Path, monkeypatch) -> None: + case = _example_case(tmp_path) + context = SimpleNamespace() + monkeypatch.setattr( + qualification_runtime, + "reference_environment_paths", + lambda *_args: {"reference_repo": "/reference/example"}, + ) + + assert reference_environment_options(case, context, {"variant": "official"}) == { + "variant": "official", + "reference_repo": "/reference/example", + } + with pytest.raises(QualificationError, match="duplicate.*reference_repo"): + reference_environment_options( + case, + context, + {"reference_repo": "/different/reference"}, + ) + + def test_qualification_summary_is_written_as_report_json(tmp_path: Path) -> None: summary = { "schema_version": "trtmc.qualification-summary/v1", @@ -74,42 +155,21 @@ def test_family_configs_auto_discover_without_a_central_model_registry() -> None assert relative.parts[1:3] == ("tests", "benchmark") assert case.kind in {"accuracy", "performance"} assert not any("l0" in case.model.lower() for case in cases) - nemotron_h = [case for case in cases if case.model == "nemotron-h-nano-9b"] - assert nemotron_h and all(not case.reference_build_isolation for case in nemotron_h) - lance = [case for case in cases if case.model == "lance-3b-x2t-image"] - assert lance and all(not case.reference_build_isolation for case in lance) - - -def test_personaplex_reference_environment_installs_official_audio_dependency( - monkeypatch, -) -> None: - calls = [] - monkeypatch.setattr(personaplex_environment, "package_version", lambda _name: "0.1.3") - monkeypatch.setattr( - personaplex_environment.subprocess, - "run", - lambda command, **kwargs: calls.append((command, kwargs)), - ) - - personaplex_environment._install_sphn() - - command, kwargs = calls[0] - assert command[-1] == "sphn==0.1.4" - assert kwargs["check"] is True - assert kwargs["env"]["CMAKE_POLICY_VERSION_MINIMUM"] == "3.5" -def test_timm_qualification_profiles_use_the_family_build_task() -> None: - cases = discover(REPOSITORY) - profiles = { - case.model: case - for case in cases - if case.kind == "accuracy" and case.family.startswith("timm_") - } +def test_qualification_profiles_match_family_manifest_tasks_when_present() -> None: + profiles = {} + for case in discover(REPOSITORY): + manifest = case.source.parent.parent / "manifests" / f"{case.model}.json" + if ( + case.kind == "accuracy" + and case.benchmark == "imagenette_classification" + and manifest.is_file() + ): + profiles[case.model] = (case, manifest) assert profiles - for model, case in profiles.items(): - manifest = case.source.parent.parent / "manifests" / f"{model}.json" + for model, (case, manifest) in profiles.items(): declared = json.loads(manifest.read_text(encoding="utf-8")) assert case.candidate["task"] == declared["task"], model @@ -389,17 +449,17 @@ def test_image_classification_accuracy_compares_top1_and_gold_accuracy( ), encoding="utf-8", ) - profile = tmp_path / "families/timm_example/tests/benchmark/example.yaml" + profile = tmp_path / "families/example/tests/benchmark/example.yaml" profile.parent.mkdir(parents=True) case = QualificationCase( kind="accuracy", model="example", - family="timm_example", + family="example", name="imagenette-parity", benchmark="imagenette_classification", candidate={ - "family": "timm_example", - "checkpoint": "timm/example", + "family": "example", + "checkpoint": "example/model", "task": "classification", "precision": "fp16", "build": {}, @@ -484,7 +544,7 @@ def reference(command, *_args, **_kwargs): def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> None: - profile = tmp_path / "families/timm_example/tests/benchmark/example.yaml" + profile = tmp_path / "families/example/tests/benchmark/example.yaml" profile.parent.mkdir(parents=True) image = profile.parent.parent / "data/test.jpeg" image.parent.mkdir(parents=True) @@ -492,12 +552,12 @@ def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> N case = QualificationCase( kind="performance", model="example", - family="timm_example", + family="example", name="classify", benchmark="image_classification_performance", candidate={ - "family": "timm_example", - "checkpoint": "timm/example", + "family": "example", + "checkpoint": "example/model", "task": "classification", "precision": "fp16", "build": {}, @@ -515,7 +575,7 @@ def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> N def test_robot_action_accuracy_compares_complete_action_chunk(tmp_path: Path, monkeypatch) -> None: - data = tmp_path / "families/lerobot_act/tests/data" + data = tmp_path / "families/example/tests/data" data.mkdir(parents=True) image = data / "image.png" state = data / "state.f32" @@ -526,19 +586,19 @@ def test_robot_action_accuracy_compares_complete_action_chunk(tmp_path: Path, mo json.dumps({"requests": [{"id": "recorded", "image": image.name, "state": state.name}]}), encoding="utf-8", ) - profile = tmp_path / "families/lerobot_act/tests/benchmark/example.yaml" + profile = tmp_path / "families/example/tests/benchmark/example.yaml" profile.parent.mkdir(parents=True) runner = profile.parent / "reference.py" runner.write_text("# family reference\n", encoding="utf-8") case = QualificationCase( kind="accuracy", - model="act-example", - family="lerobot_act", + model="example-control", + family="example", name="recorded-action-parity", benchmark="robot_action_parity", candidate={ - "family": "lerobot_act", - "checkpoint": "lerobot/example", + "family": "example", + "checkpoint": "example/control", "task": "robot_control", "precision": "fp32", "build": {}, @@ -556,7 +616,7 @@ def test_robot_action_accuracy_compares_complete_action_chunk(tmp_path: Path, mo source=profile, reference_requirements=None, ) - dataset = Dataset("lerobot-act-recorded-observation", dataset_path, "family", "digest") + dataset = Dataset("robot-action-recorded-observation", dataset_path, "family", "digest") context = RuntimeContext( repository=REPOSITORY, artifacts=tmp_path / "artifacts", @@ -863,19 +923,19 @@ def test_ocr_accuracy_uses_dataset_questions_and_reports_gold_matches( ), encoding="utf-8", ) - profile = tmp_path / "families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml" + profile = tmp_path / "families/example/tests/benchmark/example-ocr.yaml" profile.parent.mkdir(parents=True) runner = profile.parent / "reference.py" runner.write_text("# family reference\n", encoding="utf-8") case = QualificationCase( kind="accuracy", - model="deepseek-ocr", - family="deepseek_ocr", + model="example-ocr", + family="example", name="ocrbench-v2-parity", benchmark="ocrbench_v2_parity", candidate={ - "family": "deepseek_ocr", - "checkpoint": "deepseek-ai/DeepSeek-OCR-2", + "family": "example", + "checkpoint": "example/ocr", "revision": "a" * 40, "task": "vision_language_generation", "precision": "fp16", @@ -997,19 +1057,19 @@ def test_semantic_segmentation_accuracy_compares_pixel_and_class_iou( json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), encoding="utf-8", ) - profile = tmp_path / "families/segformer/tests/benchmark/example.yaml" + profile = tmp_path / "families/example/tests/benchmark/example.yaml" profile.parent.mkdir(parents=True) runner = profile.parent / "reference.py" runner.write_text("# family reference\n", encoding="utf-8") case = QualificationCase( kind="accuracy", model="example", - family="segformer", + family="example", name="segmentation-parity", benchmark="imagenette_segmentation_parity", candidate={ - "family": "segformer", - "checkpoint": "nvidia/example", + "family": "example", + "checkpoint": "example/segmentation", "task": "segmentation", "precision": "fp16", "build": {}, @@ -1136,19 +1196,19 @@ def test_image_feature_accuracy_compares_vectors_and_knn_utility( ), encoding="utf-8", ) - profile = tmp_path / "families/dinov3/tests/benchmark/example.yaml" + profile = tmp_path / "families/example/tests/benchmark/example.yaml" profile.parent.mkdir(parents=True) runner = profile.parent / "reference.py" runner.write_text("# family reference\n", encoding="utf-8") case = QualificationCase( kind="accuracy", model="example", - family="dinov3", + family="example", name="beans-knn-parity", benchmark="beans_image_feature_knn", candidate={ - "family": "dinov3", - "checkpoint": "facebook/example", + "family": "example", + "checkpoint": "example/features", "task": "image_features", "precision": "fp16", "build": {"max_sequence_length": 1}, @@ -1169,7 +1229,7 @@ def test_image_feature_accuracy_compares_vectors_and_knn_utility( source=profile, reference_requirements=None, ) - dataset = Dataset("dinov3-beans-knn-v1", dataset_path, "provided", "digest") + dataset = Dataset("image-feature-knn-v1", dataset_path, "provided", "digest") context = RuntimeContext( repository=REPOSITORY, artifacts=tmp_path / "artifacts", @@ -1530,10 +1590,10 @@ def convert_ids_to_tokens(value: int) -> str: @pytest.mark.parametrize("runner", [hf_text_generation, hf_transformers]) -def test_hf_translation_supports_transformers5_generic_nllb_tokenizer(runner) -> None: +def test_hf_translation_supports_replace_final_unknown_tokenizer(runner) -> None: import torch - class GenericNllbTokenizer: + class GenericTranslationTokenizer: unk_token_id = 3 @staticmethod @@ -1544,7 +1604,7 @@ def convert_tokens_to_ids(value: str) -> int: def convert_ids_to_tokens(value: int) -> str: return {256047: "eng_Latn", 256057: "fra_Latn"}[value] - tokenizer = GenericNllbTokenizer() + tokenizer = GenericTranslationTokenizer() request = { "source_language": "eng_Latn", "source_language_token_id": 256047, @@ -1573,16 +1633,6 @@ def convert_ids_to_tokens(value: int) -> str: assert encoded["input_ids"].tolist() == [[17, 2, 256047]] -def test_nllb_profile_owns_its_reference_source_language_placement() -> None: - cases = [case for case in discover(REPOSITORY) if case.model == "nllb-200-distilled-600m"] - - assert {case.kind for case in cases} == {"accuracy", "performance"} - assert all( - case.values["reference"]["source_language_placement"] == "replace-final-unk" - for case in cases - ) - - @pytest.mark.parametrize( ("row", "expected_status", "expected_error"), [ @@ -2724,7 +2774,7 @@ def run_reference(command, *_args, **_kwargs): def test_metric_geometry_accuracy_compares_complete_task_artifacts( tmp_path: Path, monkeypatch ) -> None: - profile = tmp_path / "families/moge/tests/benchmark/moge.yaml" + profile = tmp_path / "families/example/tests/benchmark/example-geometry.yaml" profile.parent.mkdir(parents=True) image = tmp_path / "data/image.jpeg" image.parent.mkdir() @@ -2777,13 +2827,13 @@ def geometry(root: Path) -> dict[str, object]: } case = QualificationCase( kind="accuracy", - model="moge", - family="moge", + model="example-geometry", + family="example", name="geometry", benchmark="metric_geometry_parity", candidate={ - "family": "moge", - "checkpoint": "example/moge", + "family": "example", + "checkpoint": "example/geometry", "task": "monocular_geometry", "precision": "fp32", "build": {}, @@ -2824,7 +2874,7 @@ def geometry(root: Path) -> dict[str, object]: monkeypatch.setattr( qualification_accuracy, "_candidate_outputs", - lambda *_args: ([candidate], tmp_path / "moge.bundle"), + lambda *_args: ([candidate], tmp_path / "example.bundle"), ) result = qualification_accuracy.run_accuracy(case, context) From e053d617ee969e89b0686435cb8e696e5a1ba620 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 06:56:10 +0000 Subject: [PATCH 095/100] fix(benchmark): identify TIMM reference backend Keep the release suite adapter and reported backend aligned after removing family-name dispatch from the shared runner. Signed-off-by: chaofengw --- apps/benchmark/performance/release.yaml | 56 +++++++++---------- .../trtmc_benchmark/tests/test_perf_matrix.py | 1 + 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/apps/benchmark/performance/release.yaml b/apps/benchmark/performance/release.yaml index 32ffca3d88..752c0c6172 100644 --- a/apps/benchmark/performance/release.yaml +++ b/apps/benchmark/performance/release.yaml @@ -1091,7 +1091,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_resnet.classify @@ -1104,7 +1104,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_xception.classify @@ -1117,7 +1117,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_regnet.classify @@ -1130,7 +1130,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_seresnet.classify @@ -1143,7 +1143,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_senet.classify @@ -1156,7 +1156,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_ghostnet.classify @@ -1169,7 +1169,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_convnext.classify @@ -1182,7 +1182,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_inception_v4.classify @@ -1195,7 +1195,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_inception_resnet.classify @@ -1208,7 +1208,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_hrnet.classify @@ -1221,7 +1221,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_resnest.classify @@ -1234,7 +1234,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_swin.classify @@ -1247,7 +1247,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_crossvit.classify @@ -1260,7 +1260,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_dpn.classify @@ -1273,7 +1273,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_res2net.classify @@ -1286,7 +1286,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: yolov10.detect @@ -1312,7 +1312,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilevit.classify @@ -1325,7 +1325,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: yolov8.detect @@ -1364,7 +1364,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_repvgg.classify @@ -1377,7 +1377,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_inception.classify @@ -1390,7 +1390,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mnasnet.classify @@ -1403,7 +1403,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilenetv2.classify @@ -1416,7 +1416,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_densenet.classify @@ -1429,7 +1429,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_efficientnet.classify @@ -1442,7 +1442,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilenetv4.classify @@ -1455,7 +1455,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilenetv3.classify @@ -1468,7 +1468,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_vgg.classify @@ -1481,7 +1481,7 @@ entries: runner: task-reference adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: wan_t2v.generate_image diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index b18ceaa7b8..9d4fd57a39 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -1477,6 +1477,7 @@ def test_release_suite_expands_profiles_and_covers_ready_catalog() -> None: assert builtin_timm assert all( entry["baseline"]["adapter"] == "timm-classification" + and entry["baseline"]["reference_backend"] == "timm" for entry in builtin_timm ) From c7d92eb3f55d7f717b91974db5f2b92ee1ba300b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 08:54:40 +0000 Subject: [PATCH 096/100] fix(benchmark): isolate qualification policy Select shared reference behavior only from explicit profile declarations, pin every remote qualification checkpoint to an immutable revision, and reject candidate artifact paths outside the current cell directory. Keep generic runner tests independent of concrete model families. Signed-off-by: chaofengw --- .../performance/baselines/task_reference.py | 133 +++++++++++------- apps/benchmark/performance/release.yaml | 24 ++++ .../test_diffusers_semantic_reference.py | 56 ++++++-- .../trtmc_benchmark/tests/test_perf_matrix.py | 74 +++++++++- .../albert/tests/benchmark/albert-base.yaml | 1 + families/bark/tests/benchmark/bark-large.yaml | 2 +- families/bark/tests/benchmark/bark-small.yaml | 2 +- families/bart/tests/benchmark/bart-base.yaml | 1 + .../tests/benchmark/all-minilm-l6-v2.yaml | 1 + .../tests/benchmark/bert-base-uncased.yaml | 1 + .../tests/benchmark/bge-small-en-v1.5.yaml | 1 + ...paraphrase-multilingual-minilm-l12-v2.yaml | 1 + .../canary/tests/benchmark/canary-1b-v2.yaml | 1 + .../benchmark/chronos-bolt-tiny-official.yaml | 2 + .../codegen/tests/benchmark/codegen-350m.yaml | 1 + .../tests/benchmark/convbert-base.yaml | 1 + .../deberta/tests/benchmark/deberta-base.yaml | 1 + .../tests/benchmark/deepseek-ocr.yaml | 1 + .../tests/benchmark/deepseek-v2-lite.yaml | 1 + .../detr/tests/benchmark/detr-resnet-50.yaml | 1 + ...inov3-convnext-tiny-pretrain-lvd1689m.yaml | 1 + .../dinov3-vits16-pretrain-lvd1689m.yaml | 1 + .../benchmark/distilbert-base-uncased.yaml | 1 + .../dpr/tests/benchmark/dpr-ctx-encoder.yaml | 1 + .../benchmark/electra-base-discriminator.yaml | 1 + .../flux/tests/benchmark/flux-2-dev-fp8.yaml | 1 + families/flux/tests/benchmark/flux-2-dev.yaml | 1 + .../flux/tests/benchmark/flux-schnell.yaml | 1 + families/fnet/tests/benchmark/fnet-base.yaml | 1 + .../gemma/tests/benchmark/gemma-2-2b.yaml | 1 + .../gemma/tests/benchmark/gemma-3-1b.yaml | 1 + .../gemma/tests/benchmark/gemma-3-270m.yaml | 1 + .../gemma/tests/benchmark/gemma-3-4b.yaml | 1 + families/gpt2/tests/benchmark/distilgpt2.yaml | 1 + families/gpt2/tests/benchmark/gpt2-125m.yaml | 1 + .../llama/tests/benchmark/falcon3-1b.yaml | 1 + .../tests/benchmark/nemotron-nano-4b.yaml | 1 + .../tests/benchmark/locateanything-3b.yaml | 1 + .../benchmark/nllb-200-distilled-600m.yaml | 1 + .../marian/tests/benchmark/marian-en-ru.yaml | 1 + .../mistral/tests/benchmark/mistral-7b.yaml | 1 + .../tests/benchmark/riva-translate-4b.yaml | 1 + .../tests/benchmark/modernbert-base.yaml | 1 + .../tests/benchmark/all-mpnet-base-v2.yaml | 1 + .../tests/benchmark/nemotron-mini-4b.yaml | 1 + .../nemotron-3.5-asr-streaming-0.6b.yaml | 2 + .../nemotron-speech-streaming-en-0.6b.yaml | 2 + .../patchtsmixer-granite-official.yaml | 1 + ...atchtst-etth1-regression-distribution.yaml | 1 + .../benchmark/patchtst-granite-official.yaml | 1 + families/phi/tests/benchmark/phi3-mini.yaml | 1 + families/phi_moe/tests/benchmark/phi-moe.yaml | 1 + .../tests/benchmark/pixart-sigma-1024.yaml | 1 + .../qwen/tests/benchmark/qwen3-0.6b-fp16.yaml | 1 + .../qwen/tests/benchmark/qwen3-0.6b-fp8.yaml | 1 + .../qwen/tests/benchmark/qwen3-0.6b-topp.yaml | 1 + .../benchmark/qwen3-4b-instruct-2507.yaml | 1 + .../tests/benchmark/qwen-image-2512.yaml | 4 +- .../tests/benchmark/qwen-image-edit-2511.yaml | 4 +- .../tests/benchmark/qwen-image.yaml | 4 +- .../tests/benchmark/camembert-base.yaml | 1 + .../roberta/tests/benchmark/roberta-base.yaml | 1 + .../tests/benchmark/roberta-large.yaml | 1 + .../tests/benchmark/xlm-roberta-base.yaml | 1 + .../tests/benchmark/segformer-b0-ade.yaml | 1 + .../tests/benchmark/starcoder2-3b.yaml | 1 + families/t5/tests/benchmark/t5-small.yaml | 1 + .../benchmark/timesfm-2.0-500m-official.yaml | 1 + .../tests/benchmark/densenet121-ra-in1k.yaml | 1 + .../benchmark/efficientnet-b0-ra-in1k.yaml | 1 + .../tests/benchmark/inception-v3-tv-in1k.yaml | 1 + .../benchmark/mnasnet-100-rmsp-in1k.yaml | 1 + .../benchmark/mobilenetv2-100-ra-in1k.yaml | 1 + .../mobilenetv3-large-100-ra-in1k.yaml | 1 + .../benchmark/mobilenetv4-conv-large.yaml | 1 + .../benchmark/mobilenetv4-conv-medium.yaml | 1 + .../benchmark/mobilenetv4-conv-small-050.yaml | 1 + .../benchmark/mobilenetv4-conv-small.yaml | 1 + .../tests/benchmark/resnet50-a1-in1k.yaml | 1 + .../tests/benchmark/vgg16-tv-in1k.yaml | 1 + ...vit-base-p16-224-augreg-in21k-ft-in1k.yaml | 1 + .../tests/benchmark/wan22-ti2v-5b.yaml | 2 +- .../tests/benchmark/wan21-t2v-1.3b.yaml | 2 +- .../benchmark/whisper-large-v3-turbo.yaml | 1 + .../tests/benchmark/whisper-tiny-fp16.yaml | 1 + families/xglm/tests/benchmark/xglm-564m.yaml | 1 + .../xlnet/tests/benchmark/xlnet-base.yaml | 1 + .../tests/benchmark/z-image-turbo.yaml | 2 +- tools/benchmark_qualification/accuracy.py | 22 +-- tools/benchmark_qualification/catalog.py | 6 + tools/tests/test_model_benchmark.py | 108 +++++++++++++- 91 files changed, 436 insertions(+), 88 deletions(-) diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py index a08d223b21..a94c51d70d 100644 --- a/apps/benchmark/performance/baselines/task_reference.py +++ b/apps/benchmark/performance/baselines/task_reference.py @@ -430,7 +430,7 @@ def _load_tts( torch.use_deterministic_algorithms(True) device = torch.device("cuda") prompt = str(request.get("prompt", "")) - if arguments.family == "magpie_tts": + if arguments.adapter == "nemo-tts": import fsspec from huggingface_hub import hf_hub_download from nemo.collections.tts.models import MagpieTTSModel @@ -503,7 +503,7 @@ def invoke() -> Mapping[str, Any]: "_audio_f32": audio.detach().float().cpu().reshape(-1).numpy(), } - if arguments.family == "magpie_tts": + if arguments.adapter == "nemo-tts": return Session( invoke, "nemo", @@ -515,13 +515,12 @@ def invoke() -> Mapping[str, Any]: def _load_nemo_asr_reference_model( arguments: argparse.Namespace, + options: Mapping[str, Any], *, device: Any, ) -> Any: - if ( - arguments.family == "nemotron_speech_streaming" - and "nemotron-3.5-asr-streaming" in arguments.model.lower() - ): + loader = str(options.get("model_loader", "nemo-pretrained")) + if loader == "prompted-nemo-archive": from apps.benchmark.performance.baselines.audio_reference import ( load_nemotron35_asr_model, ) @@ -532,6 +531,8 @@ def _load_nemo_asr_reference_model( local_files_only=arguments.local_files_only, device=str(device), ) + if loader != "nemo-pretrained": + raise ValueError(f"unsupported NeMo ASR model_loader: {loader}") import nemo.collections.asr as nemo_asr @@ -553,7 +554,7 @@ def _disable_nemo_asr_cuda_graphs(model: Any) -> bool: def _load_asr( arguments: argparse.Namespace, request: Mapping[str, Any], - _options: Mapping[str, Any], + options: Mapping[str, Any], ) -> Session: import torch from apps.benchmark.performance.baselines.audio_reference import ( @@ -611,8 +612,8 @@ def invoke() -> Mapping[str, Any]: "output_tokens": len(token_ids), } - elif arguments.family in {"canary", "nemotron_speech_streaming"}: - model = _load_nemo_asr_reference_model(arguments, device=device).eval().to(device) + elif arguments.adapter == "nemo-asr": + model = _load_nemo_asr_reference_model(arguments, options, device=device).eval().to(device) _disable_nemo_asr_cuda_graphs(model) reference_dtype = _torch_dtype(torch, arguments.precision) autocast_dtype = reference_dtype if arguments.precision != "fp32" else torch.float16 @@ -622,7 +623,8 @@ def invoke() -> Mapping[str, Any]: write_wav_pcm16(Path(temporary.name), audio, target_rate) transcription_input: str | list[str] - if arguments.family == "nemotron_speech_streaming": + transcription_format = str(options.get("transcription_input", "audio-files")) + if transcription_format == "manifest": manifest = tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) manifest.close() atexit.register(Path(manifest.name).unlink, missing_ok=True) @@ -636,8 +638,12 @@ def invoke() -> Mapping[str, Any]: record["lang"] = language Path(manifest.name).write_text(json.dumps(record) + "\n", encoding="utf-8") transcription_input = manifest.name - else: + elif transcription_format == "audio-files": transcription_input = [temporary.name] + else: + raise ValueError( + f"unsupported NeMo ASR transcription_input: {transcription_format}" + ) def invoke() -> Mapping[str, Any]: with torch.autocast( @@ -681,10 +687,8 @@ def invoke() -> Mapping[str, Any]: "output_tokens": len(token_ids), } - framework = ( - "nemo" if arguments.family in {"canary", "nemotron_speech_streaming"} else "transformers" - ) - if arguments.family in {"canary", "nemotron_speech_streaming"}: + framework = "nemo" if arguments.adapter == "nemo-asr" else "transformers" + if arguments.adapter == "nemo-asr": return Session( invoke, framework, @@ -696,13 +700,14 @@ def invoke() -> Mapping[str, Any]: def _load_vlm_model( - transformers_module: Any, family: str, model_id: str, kwargs: dict[str, Any] + transformers_module: Any, class_name: str, model_id: str, kwargs: dict[str, Any] ) -> Any: - class_name = { - "internvl": "AutoModel", - "phi4_multimodal": "AutoModelForCausalLM", - "qwen_vl": "AutoModelForImageTextToText", - }[family] + if class_name not in { + "AutoModel", + "AutoModelForCausalLM", + "AutoModelForImageTextToText", + }: + raise ValueError(f"unsupported vision-language model_class: {class_name}") model_class = getattr(transformers_module, class_name) return model_class.from_pretrained(model_id, **kwargs) @@ -852,12 +857,15 @@ def _vl_prompt_has_image_placeholder(text: str) -> bool: def _load_vlm( arguments: argparse.Namespace, request: Mapping[str, Any], - _options: Mapping[str, Any], + options: Mapping[str, Any], ) -> Session: - if arguments.family == "deepseek_ocr": + workflow = str(options.get("workflow", "chat")) + if workflow == "document-ocr-infer": return _load_deepseek_ocr(arguments, request) - if arguments.family == "locateanything": + if workflow == "grounded-generation": return _load_locateanything(arguments, request) + if workflow not in {"chat", "single-image-token-chat"}: + raise ValueError(f"unsupported vision-language workflow: {workflow}") import torch import transformers @@ -867,7 +875,7 @@ def _load_vlm( device = torch.device("cuda") processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) load_options = _load_kwargs(arguments, torch) - if arguments.family == "phi4_multimodal": + if bool(options.get("eager_attention", False)): config = transformers.AutoConfig.from_pretrained( arguments.model, **_processor_kwargs(arguments) ) @@ -875,7 +883,12 @@ def _load_vlm( config._attn_implementation_internal = "eager" load_options.update({"config": config, "attn_implementation": "eager"}) model = ( - _load_vlm_model(transformers, arguments.family, arguments.model, load_options) + _load_vlm_model( + transformers, + str(options.get("model_class", "AutoModelForImageTextToText")), + arguments.model, + load_options, + ) .eval() .to(device) ) @@ -883,7 +896,7 @@ def _load_vlm( prompt = str(request.get("prompt", "")) max_new_tokens = int(request.get("max_new_tokens", 16)) - if arguments.family == "phi4_multimodal": + if workflow == "single-image-token-chat": messages = [{"role": "user", "content": f"<|image_1|>{prompt}"}] template_owner = getattr(processor, "tokenizer", processor) else: @@ -1053,8 +1066,8 @@ def _diffusion_component_precision_contract( return "" if contract != _PIXART_TRTMC_MIXED_PRECISION: raise ValueError(f"unsupported diffusion component precision contract: {contract}") - if arguments.family != "pixart" or arguments.precision != "fp16": - raise ValueError(f"{contract} requires family=pixart and precision=fp16") + if arguments.precision != "fp16": + raise ValueError(f"{contract} requires precision=fp16") return contract @@ -1129,11 +1142,19 @@ def _diffusion_pipeline( ), local_files_only=arguments.local_files_only, ) - if arguments.family == "wan2_2_ti2v": - load_options["vae"] = diffusers.AutoencoderKLWan.from_pretrained( + vae_class_name = options.get("vae_class") + if vae_class_name is not None: + if not isinstance(vae_class_name, str) or not vae_class_name.startswith("Autoencoder"): + raise ValueError("diffusion adapter_options.vae_class must name an Autoencoder class") + try: + vae_class = getattr(diffusers, vae_class_name) + except AttributeError as error: + raise ValueError(f"unsupported Diffusers VAE class: {vae_class_name}") from error + vae_precision = str(options.get("vae_precision", arguments.precision)) + load_options["vae"] = vae_class.from_pretrained( model_source, subfolder="vae", - torch_dtype=torch_module.float32, + torch_dtype=_torch_dtype(torch_module, vae_precision), **( {"revision": requested_revision} if requested_revision and model_source == model_id @@ -1200,7 +1221,11 @@ def _load_diffusers( prompt_value = [prompt] * batch_size if batch_size > 1 else prompt values: dict[str, Any] = {"prompt": prompt_value} negative_prompt = str(request.get("negative_prompt", "")) - if negative_prompt or "negative_prompt" in request or arguments.family == "qwen_image": + if ( + negative_prompt + or "negative_prompt" in request + or bool(options.get("always_negative_prompt", False)) + ): values["negative_prompt"] = negative_prompt steps = int(request.get("num_steps", -1)) if steps > 0: @@ -1234,13 +1259,15 @@ def _load_diffusers( if value is not None: values[name] = float(value) cfg_scale = float(request.get("cfg_scale", -1.0)) - if cfg_scale >= 0 and arguments.family != "qwen_image": + guidance_parameter = str(options.get("guidance_parameter", "guidance_scale")) + if guidance_parameter not in {"guidance_scale", "true_cfg_scale"}: + raise ValueError(f"unsupported diffusion guidance_parameter: {guidance_parameter}") + if cfg_scale >= 0 and guidance_parameter != "true_cfg_scale": values["cfg_scale"] = cfg_scale if bool(request.get("no_refiner", False)): values["no_refiner"] = True guidance = float(request.get("guidance_scale", -1.0)) - if arguments.family == "qwen_image": - # Native Qwen guidance is true CFG; cfg_scale remains a legacy fallback. + if guidance_parameter == "true_cfg_scale": true_cfg_scale = guidance if guidance >= 0 else cfg_scale if true_cfg_scale >= 0: values["true_cfg_scale"] = true_cfg_scale @@ -1546,7 +1573,7 @@ def _patchtst_task(config: Any) -> str: def _load_timeseries( arguments: argparse.Namespace, request: Mapping[str, Any], - _options: Mapping[str, Any], + options: Mapping[str, Any], ) -> Session: import torch import transformers @@ -1554,7 +1581,8 @@ def _load_timeseries( device = torch.device("cuda") dtype = _torch_dtype(torch, arguments.precision) task_id = _selected_task(arguments) - if arguments.family == "chronos_bolt": + reference_type = str(options.get("reference_type", "")) + if reference_type == "chronos-bolt": from chronos import ChronosBoltPipeline chronos_options = _processor_kwargs(arguments) @@ -1586,7 +1614,7 @@ def invoke() -> Mapping[str, Any]: config = transformers.AutoConfig.from_pretrained( arguments.model, **_processor_kwargs(arguments) ) - if arguments.family == "timesfm": + if reference_type == "timesfm": model = ( transformers.TimesFmModelForPrediction.from_pretrained( arguments.model, **_load_kwargs(arguments, torch) @@ -1620,7 +1648,9 @@ def invoke() -> Mapping[str, Any]: return _forecast_summary(output, task_id) else: - is_mixer = arguments.family == "patchtsmixer" + if reference_type not in {"patchtsmixer", "patchtst"}: + raise ValueError(f"unsupported time-series reference_type: {reference_type}") + is_mixer = reference_type == "patchtsmixer" if is_mixer: model_class = transformers.PatchTSMixerForPrediction output_name = "prediction_outputs" @@ -1695,7 +1725,7 @@ def invoke() -> Mapping[str, Any]: def _load_vision( arguments: argparse.Namespace, request: Mapping[str, Any], - _options: Mapping[str, Any], + options: Mapping[str, Any], ) -> Session: import torch from PIL import Image @@ -1706,6 +1736,7 @@ def _load_vision( width, height = image.size kwargs = _load_kwargs(arguments, torch) processor_kwargs = _processor_kwargs(arguments) + vision_task = str(options.get("vision_task", "")) if arguments.adapter == "timm-classification": import timm @@ -1725,7 +1756,7 @@ def invoke() -> Mapping[str, Any]: logits = model(inputs) return {"top_class": int(logits.argmax(dim=-1)[0]), **_tensor_summary(logits)} - elif arguments.family == "detr": + elif vision_task == "object-detection": processor = transformers.AutoImageProcessor.from_pretrained( arguments.model, **processor_kwargs ) @@ -1760,7 +1791,7 @@ def invoke() -> Mapping[str, Any]: "units": "pixels", } - elif arguments.family == "dinov3": + elif vision_task == "image-features": processor = transformers.AutoImageProcessor.from_pretrained( arguments.model, **processor_kwargs ) @@ -1779,7 +1810,7 @@ def invoke() -> Mapping[str, Any]: "pooler_output_shape": _tensor_summary(outputs.pooler_output)["shape"], } - elif arguments.family == "segformer": + elif vision_task == "semantic-segmentation": processor = transformers.AutoImageProcessor.from_pretrained( arguments.model, **processor_kwargs ) @@ -1806,7 +1837,7 @@ def invoke() -> Mapping[str, Any]: "width": int(mask.shape[1]), } - elif arguments.family == "sam3": + elif vision_task == "text-prompted-segmentation": processor = _load_sam3_processor(transformers, arguments.model, processor_kwargs) model = transformers.Sam3Model.from_pretrained(arguments.model, **kwargs).eval().to(device) inputs = _to_device( @@ -1840,7 +1871,7 @@ def invoke() -> Mapping[str, Any]: "width": width, } - else: + elif vision_task == "point-prompted-segmentation": processor = transformers.SamProcessor.from_pretrained(arguments.model, **processor_kwargs) model = transformers.SamModel.from_pretrained(arguments.model, **kwargs).eval().to(device) points = [ @@ -1873,6 +1904,9 @@ def invoke() -> Mapping[str, Any]: "width": width, } + else: + raise ValueError(f"unsupported Transformers vision_task: {vision_task}") + framework = "timm" if arguments.adapter == "timm-classification" else "transformers" return Session(invoke, framework) @@ -3020,16 +3054,19 @@ def _environment() -> dict[str, Any]: def run(arguments: argparse.Namespace) -> int: if arguments.warmup < 0 or arguments.iterations <= 0: raise ValueError("warmup must be non-negative and iterations must be positive") + options = _json_object(arguments.adapter_options_json, "--adapter-options-json") expected_mode = "pytorch-eager" if arguments.adapter in PYTORCH_ADAPTERS else "hf-eager" supported_modes = {expected_mode} - if arguments.adapter == "pytorch-timeseries" and arguments.family == "chronos_bolt": + if ( + arguments.adapter == "pytorch-timeseries" + and options.get("reference_type") == "chronos-bolt" + ): supported_modes.add("torch-compile") if arguments.adapter == "hf-transformers-embedding": supported_modes.add("torch-compile") if arguments.mode not in supported_modes: raise ValueError(f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}") request = flatten_config(_json_object(arguments.request_json, "--request-json")) - options = _json_object(arguments.adapter_options_json, "--adapter-options-json") configured_timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") fields = ("timing_scope", "input_preparation_included", "asset_loading_included") expected_timing = None diff --git a/apps/benchmark/performance/release.yaml b/apps/benchmark/performance/release.yaml index 752c0c6172..fcb7410073 100644 --- a/apps/benchmark/performance/release.yaml +++ b/apps/benchmark/performance/release.yaml @@ -148,6 +148,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: object-detection} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -246,6 +247,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: chronos-bolt} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -290,6 +292,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {workflow: document-ocr-infer} mode: hf-eager reference_backend: hf_transformers timing_scope: task-pipeline-call-wall @@ -323,6 +326,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: image-features} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -541,6 +545,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {model_class: AutoModel} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -603,6 +608,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {workflow: grounded-generation} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -742,6 +748,7 @@ entries: baseline: runner: task-reference adapter: nemo-asr + adapter_options: {transcription_input: manifest, model_loader: prompted-nemo-archive} mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -787,6 +794,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtsmixer} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -800,6 +808,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtst} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -839,6 +848,10 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: + model_class: AutoModelForCausalLM + workflow: single-image-token-chat + eager_attention: true mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -932,6 +945,8 @@ entries: output_contract: media-shape adapter_options: cpu_offload: true + always_negative_prompt: true + guidance_parameter: true_cfg_scale - id: qwen_moe.generate family: qwen_moe operation: generate @@ -952,6 +967,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {model_class: AutoModelForImageTextToText} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -985,6 +1001,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: point-prompted-segmentation} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -999,6 +1016,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: text-prompted-segmentation} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1032,6 +1050,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: semantic-segmentation} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1077,6 +1096,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: timesfm} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -1515,6 +1535,8 @@ entries: adapter_options: model_id: Wan-AI/Wan2.2-TI2V-5B-Diffusers model_revision: b8fff7315c768468a5333511427288870b2e9635 + vae_class: AutoencoderKLWan + vae_precision: fp32 - id: whisper.transcribe family: whisper operation: transcribe @@ -1599,6 +1621,8 @@ additional_profiles: inherit: llama.generate - model: nemotron-speech-streaming-en-0.6b inherit: nemotron_speech_streaming.transcribe + baseline: + adapter_options: {transcription_input: manifest, model_loader: nemo-pretrained} - model: paraphrase-multilingual-minilm-l12-v2 inherit: bert.encode - model: patchtst-granite-official diff --git a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py b/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py index 49aab38b1a..19f67a0757 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py @@ -220,12 +220,20 @@ def test_invalid_batch_requests_do_not_call_pipeline(recording_diffusers, payloa @pytest.mark.parametrize("class_name", ["QwenImageEditPipeline", "QwenImageEditPlusPipeline"]) def test_qwen_edit_checkpoint_selects_edit_class_and_receives_full_request(recording_diffusers, tmp_path, image_key, class_name): recorded = recording_diffusers - arguments = recorded.arguments(class_name, "qwen_image", "images_text_to_image_edit") + arguments = recorded.arguments(class_name, "synthetic", "images_text_to_image_edit") image = tmp_path / "condition.ppm" image.write_bytes(b"P6\n1 1\n255\n\xff\x00\x80") request = {"prompt": "Make it blue", image_key: [image.name] if image_key == "image_paths" else image.name, "config": {"seed": 2**32 - 1, "num_steps": 5, "guidance_scale": 3.25, "negative_prompt": " "}} - session = task_reference._load_diffusers(arguments, task_reference.flatten_config(request), {"cpu_offload": True}) + session = task_reference._load_diffusers( + arguments, + task_reference.flatten_config(request), + { + "cpu_offload": True, + "always_negative_prompt": True, + "guidance_parameter": "true_cfg_scale", + }, + ) session.invoke() pipeline = recorded.constructors[0] assert pipeline.__class__.__name__ == class_name @@ -256,8 +264,12 @@ def test_qwen_edit_checkpoint_selects_edit_class_and_receives_full_request(recor @pytest.mark.parametrize("negative", [{}, {"negative_prompt": ""}]) def test_qwen_native_guidance_controls_true_cfg_and_preserves_empty_negative_prompt(recording_diffusers, controls, expected, negative): recorded = recording_diffusers - arguments = recorded.arguments("QwenImagePipeline", "qwen_image") - session = task_reference._load_diffusers(arguments, {"prompt": "a", **negative, **controls}, {}) + arguments = recorded.arguments("QwenImagePipeline", "synthetic") + session = task_reference._load_diffusers( + arguments, + {"prompt": "a", **negative, **controls}, + {"always_negative_prompt": True, "guidance_parameter": "true_cfg_scale"}, + ) session.invoke() assert recorded.calls[0]["true_cfg_scale"] == expected assert recorded.calls[0]["guidance_scale"] is None @@ -280,7 +292,9 @@ def test_non_qwen_guidance_retains_provider_parameter(recording_diffusers): ]) def test_qwen_edit_image_contract_fails_before_invocation(recording_diffusers, inputs, match): recorded = recording_diffusers - arguments = recorded.arguments("QwenImageEditPlusPipeline", "qwen_image", "images_text_to_image_edit") + arguments = recorded.arguments( + "QwenImageEditPlusPipeline", "synthetic", "images_text_to_image_edit" + ) with pytest.raises(ValueError, match=match): task_reference._load_diffusers(arguments, {"prompt": "edit", **inputs}, {}) assert recorded.calls == recorded.images == [] @@ -288,23 +302,39 @@ def test_qwen_edit_image_contract_fails_before_invocation(recording_diffusers, i def test_conditioning_image_is_never_silently_filtered(recording_diffusers): recorded = recording_diffusers - arguments = recorded.arguments("QwenImagePipeline", "qwen_image", "images_text_to_image_edit") + arguments = recorded.arguments("QwenImagePipeline", "synthetic", "images_text_to_image_edit") with pytest.raises(ValueError, match="does not accept conditioning image"): task_reference._load_diffusers(arguments, {"prompt": "edit", "image_paths": ["condition.ppm"]}, {}) assert recorded.calls == [] -@pytest.mark.parametrize("family,class_name,contract,component_key,subfolder", [ - ("pixart", "PixArtSigmaPipeline", "pixart_fp16_dit_fp32_t5", "text_encoder", "text_encoder"), - ("wan2_2_ti2v", "WanPipeline", "", "vae", "vae"), +@pytest.mark.parametrize("class_name,options,component_key,subfolder", [ + ( + "PixArtSigmaPipeline", + {"component_precision_contract": "pixart_fp16_dit_fp32_t5"}, + "text_encoder", + "text_encoder", + ), + ( + "WanPipeline", + {"vae_class": "AutoencoderKLWan", "vae_precision": "fp32"}, + "vae", + "vae", + ), ]) @pytest.mark.parametrize("local_files_only", [True, False]) def test_checkpoint_owned_loader_preserves_component_precision_and_revision( - recording_diffusers, family, class_name, contract, component_key, subfolder, local_files_only, + recording_diffusers, class_name, options, component_key, subfolder, local_files_only, ): recorded = recording_diffusers - arguments = recorded.arguments(class_name, family, precision="fp16", local_files_only=local_files_only) - options = {"component_precision_contract": contract, "model_revision": "override-revision", "trust_remote_code": True} + arguments = recorded.arguments( + class_name, "synthetic", precision="fp16", local_files_only=local_files_only + ) + options = { + **options, + "model_revision": "override-revision", + "trust_remote_code": True, + } session = task_reference._load_diffusers(arguments, {"prompt": "a"}, options) session.invoke() source, kwargs, component = recorded.components[0] @@ -316,7 +346,7 @@ def test_checkpoint_owned_loader_preserves_component_precision_and_revision( "torch_dtype": "fp16", component_key: component, "local_files_only": local_files_only, "trust_remote_code": True, **revision})] assert recorded.constructors[0].__class__.__name__ == class_name - if contract: + if options.get("component_precision_contract"): transformer = recorded.constructors[0].transformer assert len(transformer.pre_hooks) == len(transformer.hooks) == 1 assert transformer.pre_hooks[0][1] == {"with_kwargs": True} diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py index 9d4fd57a39..d042fc2997 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py @@ -3,6 +3,7 @@ from __future__ import annotations +import ast import sys from contextlib import nullcontext from copy import deepcopy @@ -23,12 +24,40 @@ sana_wm_reference, task_reference, ) +from apps.benchmark.trtmc_benchmark.types import ModelDescriptor REPO = Path(__file__).resolve().parents[4] SUITE = REPO / "apps/benchmark/performance/release.yaml" +def test_shared_reference_runner_never_uses_family_identity_to_select_behavior() -> None: + tree = ast.parse(Path(task_reference.__file__).read_text(encoding="utf-8")) + parents: dict[ast.AST, ast.AST] = { + child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent) + } + family_reads = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and node.attr == "family" + and isinstance(node.value, ast.Name) + and node.value.id == "arguments" + ] + controls = [ + node.test + for node in ast.walk(tree) + if isinstance(node, (ast.If, ast.IfExp)) + ] + [node.subject for node in ast.walk(tree) if isinstance(node, ast.Match)] + violations = [ + node.lineno + for node in family_reads + if any(node in ast.walk(control) for control in controls) + or (isinstance(parents.get(node), ast.Call) and node in parents[node].args) + ] + assert violations == [] + + def test_reference_config_preserves_explicit_values_without_defaults() -> None: original = { "source_text": "Hello", @@ -235,15 +264,50 @@ def generate(self, **kwargs): def test_reference_only_source_language_placement_reaches_hf_runner( - tmp_path: Path, + tmp_path: Path, monkeypatch, ) -> None: _, environment = _environment(tmp_path) - _, entries, _ = perf.load_suite(SUITE) - original = next(row for row in entries if row["id"] == "m2m_100.generate") + manifest = tmp_path / "synthetic" / "tests" / "manifest.json" + manifest.parent.mkdir(parents=True) + manifest.write_text( + json.dumps({"task": "text_translation", "reference_precision": "fp16"}), + encoding="utf-8", + ) + model = ModelDescriptor( + name="synthetic-translator", + hf_id="example/synthetic-translator", + hf_revision="a" * 40, + bundle_name="synthetic-translator.bundle", + family="synthetic", + task="text_translation", + precision="fp16", + manifest_path=manifest, + testcases=( + { + "name": "translate", + "source_text": "Hello", + "source_language": "eng_Latn", + "target_language": "fra_Latn", + "max_new_tokens": 8, + }, + ), + build_settings={"max_sequence_length": 32}, + ) + monkeypatch.setattr(perf.ManifestCatalog, "resolve", lambda *_args: model) spec = { - **original, + "id": "synthetic.translate", + "family": "synthetic", + "operation": "translate", + "model": model.name, + "workload": {"testcase": "translate"}, + "measurement": {"warmup": 1, "iterations": 1}, "baseline": { - **original["baseline"], + "runner": "hf-transformers", + "task": "seq2seq-lm", + "mode": "hf-eager", + "timing_scope": "public_operation_call_wall", + "input_preparation_included": True, + "asset_loading_included": False, "source_language_placement": "replace-final-unk", }, } diff --git a/families/albert/tests/benchmark/albert-base.yaml b/families/albert/tests/benchmark/albert-base.yaml index 62ad5286dc..b8045f41f9 100644 --- a/families/albert/tests/benchmark/albert-base.yaml +++ b/families/albert/tests/benchmark/albert-base.yaml @@ -6,6 +6,7 @@ model: albert-base candidate: family: albert checkpoint: albert/albert-base-v2 + revision: 8e2f239c5f8a2c0f253781ca60135db913e5c80c task: encoding precision: fp16 bundle: albert-base.bundle diff --git a/families/bark/tests/benchmark/bark-large.yaml b/families/bark/tests/benchmark/bark-large.yaml index f48edd2a50..c9f726b3f2 100644 --- a/families/bark/tests/benchmark/bark-large.yaml +++ b/families/bark/tests/benchmark/bark-large.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: bark-large -candidate: {family: bark, checkpoint: suno/bark, task: audio_generation, precision: fp32, bundle: bark-large.bundle, build: {max_sequence_length: 1024}} +candidate: {family: bark, checkpoint: suno/bark, revision: 70a8a7d34168586dc5d028fa9666aceade177992, task: audio_generation, precision: fp32, bundle: bark-large.bundle, build: {max_sequence_length: 1024}} reference_environment: {requirements: ../../requirements.txt} accuracy: - name: generated-audio-parity diff --git a/families/bark/tests/benchmark/bark-small.yaml b/families/bark/tests/benchmark/bark-small.yaml index 9bee42731d..386052b15b 100644 --- a/families/bark/tests/benchmark/bark-small.yaml +++ b/families/bark/tests/benchmark/bark-small.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: bark-small -candidate: {family: bark, checkpoint: suno/bark-small, task: audio_generation, precision: fp16, bundle: bark-small.bundle, build: {max_sequence_length: 1024}} +candidate: {family: bark, checkpoint: suno/bark-small, revision: 1dbd7a128513b8ae4a4e2130fed57b7ac9da5bcd, task: audio_generation, precision: fp16, bundle: bark-small.bundle, build: {max_sequence_length: 1024}} reference_environment: {requirements: ../../requirements.txt} accuracy: - name: generated-audio-parity diff --git a/families/bart/tests/benchmark/bart-base.yaml b/families/bart/tests/benchmark/bart-base.yaml index ab5d2872a2..a4d35b3f14 100644 --- a/families/bart/tests/benchmark/bart-base.yaml +++ b/families/bart/tests/benchmark/bart-base.yaml @@ -6,6 +6,7 @@ model: bart-base candidate: family: bart checkpoint: facebook/bart-base + revision: aadd2ab0ae0c8268c7c9693540e9904811f36177 task: text_generation precision: fp16 bundle: bart-base.bundle diff --git a/families/bert/tests/benchmark/all-minilm-l6-v2.yaml b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml index a6f5ee84a5..3d7ec772c8 100644 --- a/families/bert/tests/benchmark/all-minilm-l6-v2.yaml +++ b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml @@ -6,6 +6,7 @@ model: all-minilm-l6-v2 candidate: family: bert checkpoint: sentence-transformers/all-MiniLM-L6-v2 + revision: 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 task: embedding precision: fp16 bundle: all-minilm-l6-v2.bundle diff --git a/families/bert/tests/benchmark/bert-base-uncased.yaml b/families/bert/tests/benchmark/bert-base-uncased.yaml index 2106df7abf..e99f8a34ec 100644 --- a/families/bert/tests/benchmark/bert-base-uncased.yaml +++ b/families/bert/tests/benchmark/bert-base-uncased.yaml @@ -6,6 +6,7 @@ model: bert-base-uncased candidate: family: bert checkpoint: google-bert/bert-base-uncased + revision: 86b5e0934494bd15c9632b12f734a8a67f723594 task: encoding precision: fp16 bundle: bert-base-uncased.bundle diff --git a/families/bert/tests/benchmark/bge-small-en-v1.5.yaml b/families/bert/tests/benchmark/bge-small-en-v1.5.yaml index ba2ab50f22..cdf998a260 100644 --- a/families/bert/tests/benchmark/bge-small-en-v1.5.yaml +++ b/families/bert/tests/benchmark/bge-small-en-v1.5.yaml @@ -6,6 +6,7 @@ model: bge-small-en-v1.5 candidate: family: bert checkpoint: BAAI/bge-small-en-v1.5 + revision: 5c38ec7c405ec4b44b94cc5a9bb96e735b38267a task: encoding precision: fp16 bundle: bge-small-en-v1.5.bundle diff --git a/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml b/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml index 359ca090c1..d5010f9bf2 100644 --- a/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml +++ b/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml @@ -6,6 +6,7 @@ model: paraphrase-multilingual-minilm-l12-v2 candidate: family: bert checkpoint: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + revision: e8f8c211226b894fcb81acc59f3b34ba3efd5f42 task: encoding precision: fp16 bundle: paraphrase-multilingual-minilm-l12-v2.bundle diff --git a/families/canary/tests/benchmark/canary-1b-v2.yaml b/families/canary/tests/benchmark/canary-1b-v2.yaml index 5cbef36fa0..f526943067 100644 --- a/families/canary/tests/benchmark/canary-1b-v2.yaml +++ b/families/canary/tests/benchmark/canary-1b-v2.yaml @@ -6,6 +6,7 @@ model: canary-1b-v2 candidate: family: canary checkpoint: nvidia/canary-1b-v2 + revision: d455706339a6b32e1aa40f82c713a482a0c938e2 task: transcription precision: fp16 bundle: canary-1b-v2.bundle diff --git a/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml b/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml index a2df47f216..bc2f31568a 100644 --- a/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml +++ b/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml @@ -7,6 +7,7 @@ model: chronos-bolt-tiny-official candidate: family: chronos_bolt checkpoint: amazon/chronos-bolt-tiny + revision: a0e552de83495b5c28c14c71c374f3e33280b340 task: time_series_forecast precision: fp32 bundle: chronos-bolt-tiny-official.bundle @@ -57,6 +58,7 @@ performance: reference: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: chronos-bolt} mode: torch-compile fallback: pytorch-eager precision: fp32 diff --git a/families/codegen/tests/benchmark/codegen-350m.yaml b/families/codegen/tests/benchmark/codegen-350m.yaml index f4dc7edb5e..d7c61e6a73 100644 --- a/families/codegen/tests/benchmark/codegen-350m.yaml +++ b/families/codegen/tests/benchmark/codegen-350m.yaml @@ -6,6 +6,7 @@ model: codegen-350m candidate: family: codegen checkpoint: Salesforce/codegen-350M-mono + revision: d9107f71cca463240db1143f4a75a927a27fcb27 task: text_generation precision: fp16 bundle: codegen-350m.bundle diff --git a/families/convbert/tests/benchmark/convbert-base.yaml b/families/convbert/tests/benchmark/convbert-base.yaml index a2ae194bee..2c34168342 100644 --- a/families/convbert/tests/benchmark/convbert-base.yaml +++ b/families/convbert/tests/benchmark/convbert-base.yaml @@ -6,6 +6,7 @@ model: convbert-base candidate: family: convbert checkpoint: YituTech/conv-bert-base + revision: 5cb451936b5c4a96562d8b146de85f64f9cf2c22 task: encoding precision: fp16 bundle: convbert-base.bundle diff --git a/families/deberta/tests/benchmark/deberta-base.yaml b/families/deberta/tests/benchmark/deberta-base.yaml index 352cd94941..87adcb67f4 100644 --- a/families/deberta/tests/benchmark/deberta-base.yaml +++ b/families/deberta/tests/benchmark/deberta-base.yaml @@ -6,6 +6,7 @@ model: deberta-base candidate: family: deberta checkpoint: microsoft/deberta-base + revision: 0d1b43ccf21b5acd9f4e5f7b077fa698f05cf195 task: encoding precision: fp16 bundle: deberta-base.bundle diff --git a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml index f6f6817b3f..b8f21ef425 100644 --- a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml +++ b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml @@ -49,6 +49,7 @@ performance: reference: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {workflow: document-ocr-infer} mode: hf-eager reference_backend: hf_transformers precision: bf16 diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml index e0a13a0a18..423fdde74e 100644 --- a/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml @@ -6,6 +6,7 @@ model: deepseek-v2-lite candidate: family: deepseek_v2 checkpoint: deepseek-ai/DeepSeek-V2-Lite + revision: 604d5664dddd88a0433dbae533b7fe9472482de0 task: text_generation precision: fp16 bundle: deepseek-v2-lite.bundle diff --git a/families/detr/tests/benchmark/detr-resnet-50.yaml b/families/detr/tests/benchmark/detr-resnet-50.yaml index 1d7b65caa4..6b2c77093c 100644 --- a/families/detr/tests/benchmark/detr-resnet-50.yaml +++ b/families/detr/tests/benchmark/detr-resnet-50.yaml @@ -45,6 +45,7 @@ performance: reference: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: object-detection} mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml index b8b69a46ac..aaee48a41f 100644 --- a/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml +++ b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml @@ -42,6 +42,7 @@ performance: reference: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: image-features} mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml index 8fd63e347f..58a866c8d7 100644 --- a/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml +++ b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml @@ -42,6 +42,7 @@ performance: reference: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: image-features} mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml b/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml index 0358cd2e6b..4d10a25223 100644 --- a/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml +++ b/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml @@ -6,6 +6,7 @@ model: distilbert-base-uncased candidate: family: distilbert checkpoint: distilbert/distilbert-base-uncased + revision: 12040accade4e8a0f71eabdb258fecc2e7e948be task: encoding precision: fp16 bundle: distilbert-base-uncased.bundle diff --git a/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml index 1d5ec2a480..cf2d0c9807 100644 --- a/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml +++ b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml @@ -6,6 +6,7 @@ model: dpr-ctx-encoder candidate: family: dpr checkpoint: facebook/dpr-ctx_encoder-single-nq-base + revision: bb21a3c2b1656d60c6a8e920283bc40dabddadb8 task: encoding precision: fp16 bundle: dpr-ctx-encoder.bundle diff --git a/families/electra/tests/benchmark/electra-base-discriminator.yaml b/families/electra/tests/benchmark/electra-base-discriminator.yaml index c711b0c926..22b4ecf591 100644 --- a/families/electra/tests/benchmark/electra-base-discriminator.yaml +++ b/families/electra/tests/benchmark/electra-base-discriminator.yaml @@ -6,6 +6,7 @@ model: electra-base-discriminator candidate: family: electra checkpoint: google/electra-base-discriminator + revision: 1ae76a97c7e84a4e640876a07453fccd636f0667 task: encoding precision: fp16 bundle: electra-base-discriminator.bundle diff --git a/families/flux/tests/benchmark/flux-2-dev-fp8.yaml b/families/flux/tests/benchmark/flux-2-dev-fp8.yaml index abdf074931..ecddccfed2 100644 --- a/families/flux/tests/benchmark/flux-2-dev-fp8.yaml +++ b/families/flux/tests/benchmark/flux-2-dev-fp8.yaml @@ -5,6 +5,7 @@ model: flux-2-dev-fp8 candidate: family: flux checkpoint: black-forest-labs/FLUX.2-dev + revision: 26afe3a78bb242c0a8bb181dcc8937bb16e5c66c task: image_generation precision: fp16 bundle: flux-2-dev-fp8.bundle diff --git a/families/flux/tests/benchmark/flux-2-dev.yaml b/families/flux/tests/benchmark/flux-2-dev.yaml index 3352be03ff..15258d13da 100644 --- a/families/flux/tests/benchmark/flux-2-dev.yaml +++ b/families/flux/tests/benchmark/flux-2-dev.yaml @@ -5,6 +5,7 @@ model: flux-2-dev candidate: family: flux checkpoint: black-forest-labs/FLUX.2-dev + revision: 26afe3a78bb242c0a8bb181dcc8937bb16e5c66c task: image_generation precision: bf16 bundle: flux-2-dev.bundle diff --git a/families/flux/tests/benchmark/flux-schnell.yaml b/families/flux/tests/benchmark/flux-schnell.yaml index 27e3a7be3a..c15b8e36f4 100644 --- a/families/flux/tests/benchmark/flux-schnell.yaml +++ b/families/flux/tests/benchmark/flux-schnell.yaml @@ -5,6 +5,7 @@ model: flux-schnell candidate: family: flux checkpoint: black-forest-labs/FLUX.1-schnell + revision: 741f7c3ce8b383c54771c7003378a50191e9efe9 task: image_generation precision: fp16 bundle: flux-schnell.bundle diff --git a/families/fnet/tests/benchmark/fnet-base.yaml b/families/fnet/tests/benchmark/fnet-base.yaml index 33da47203e..4a32f80d6b 100644 --- a/families/fnet/tests/benchmark/fnet-base.yaml +++ b/families/fnet/tests/benchmark/fnet-base.yaml @@ -6,6 +6,7 @@ model: fnet-base candidate: family: fnet checkpoint: google/fnet-base + revision: d89b6fad3cf5384848b783dc480f9685f49d008c task: encoding precision: fp16 bundle: fnet-base.bundle diff --git a/families/gemma/tests/benchmark/gemma-2-2b.yaml b/families/gemma/tests/benchmark/gemma-2-2b.yaml index 954bdc8629..b70117cd12 100644 --- a/families/gemma/tests/benchmark/gemma-2-2b.yaml +++ b/families/gemma/tests/benchmark/gemma-2-2b.yaml @@ -6,6 +6,7 @@ model: gemma-2-2b candidate: family: gemma checkpoint: google/gemma-2-2b-it + revision: 299a8560bedf22ed1c72a8a11e7dce4a7f9f51f8 task: text_generation precision: fp16 bundle: gemma-2-2b.bundle diff --git a/families/gemma/tests/benchmark/gemma-3-1b.yaml b/families/gemma/tests/benchmark/gemma-3-1b.yaml index 331168f26b..0c2e2dc843 100644 --- a/families/gemma/tests/benchmark/gemma-3-1b.yaml +++ b/families/gemma/tests/benchmark/gemma-3-1b.yaml @@ -6,6 +6,7 @@ model: gemma-3-1b candidate: family: gemma checkpoint: google/gemma-3-1b-it + revision: dcc83ea841ab6100d6b47a070329e1ba4cf78752 task: text_generation precision: bf16 bundle: gemma-3-1b.bundle diff --git a/families/gemma/tests/benchmark/gemma-3-270m.yaml b/families/gemma/tests/benchmark/gemma-3-270m.yaml index 353fceb252..eaca7fbb04 100644 --- a/families/gemma/tests/benchmark/gemma-3-270m.yaml +++ b/families/gemma/tests/benchmark/gemma-3-270m.yaml @@ -6,6 +6,7 @@ model: gemma-3-270m candidate: &candidate family: gemma checkpoint: google/gemma-3-270m-it + revision: ac82b4e820549b854eebf28ce6dedaf9fdfa17b3 task: text_generation precision: bf16 bundle: gemma-3-270m.bundle diff --git a/families/gemma/tests/benchmark/gemma-3-4b.yaml b/families/gemma/tests/benchmark/gemma-3-4b.yaml index 2b2df5a2e2..0b6d94cc70 100644 --- a/families/gemma/tests/benchmark/gemma-3-4b.yaml +++ b/families/gemma/tests/benchmark/gemma-3-4b.yaml @@ -6,6 +6,7 @@ model: gemma-3-4b candidate: family: gemma checkpoint: google/gemma-3-4b-it + revision: 093f9f388b31de276ce2de164bdc2081324b9767 task: text_generation precision: bf16 bundle: gemma-3-4b.bundle diff --git a/families/gpt2/tests/benchmark/distilgpt2.yaml b/families/gpt2/tests/benchmark/distilgpt2.yaml index cd014ca772..221c6e469c 100644 --- a/families/gpt2/tests/benchmark/distilgpt2.yaml +++ b/families/gpt2/tests/benchmark/distilgpt2.yaml @@ -6,6 +6,7 @@ model: distilgpt2 candidate: family: gpt2 checkpoint: distilbert/distilgpt2 + revision: 2290a62682d06624634c1f46a6ad5be0f47f38aa task: text_generation precision: fp16 bundle: distilgpt2.bundle diff --git a/families/gpt2/tests/benchmark/gpt2-125m.yaml b/families/gpt2/tests/benchmark/gpt2-125m.yaml index 447178191c..cc920726ff 100644 --- a/families/gpt2/tests/benchmark/gpt2-125m.yaml +++ b/families/gpt2/tests/benchmark/gpt2-125m.yaml @@ -7,6 +7,7 @@ model: gpt2-125m candidate: family: gpt2 checkpoint: openai-community/gpt2 + revision: 607a30d783dfa663caf39e06633721c8d4cfcd7e task: text_generation precision: fp32 bundle: gpt2-125m.bundle diff --git a/families/llama/tests/benchmark/falcon3-1b.yaml b/families/llama/tests/benchmark/falcon3-1b.yaml index 15ea04715f..66d7e02d18 100644 --- a/families/llama/tests/benchmark/falcon3-1b.yaml +++ b/families/llama/tests/benchmark/falcon3-1b.yaml @@ -6,6 +6,7 @@ model: falcon3-1b candidate: family: llama checkpoint: tiiuae/Falcon3-1B-Base + revision: cb37ef3559b157b5c9d9226296ba01a5162da1f7 task: text_generation precision: fp16 bundle: falcon3-1b.bundle diff --git a/families/llama/tests/benchmark/nemotron-nano-4b.yaml b/families/llama/tests/benchmark/nemotron-nano-4b.yaml index 058997273e..5c27961e07 100644 --- a/families/llama/tests/benchmark/nemotron-nano-4b.yaml +++ b/families/llama/tests/benchmark/nemotron-nano-4b.yaml @@ -6,6 +6,7 @@ model: nemotron-nano-4b candidate: family: llama checkpoint: nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1 + revision: d552708a9d575fa8d4a690b988fd870d65279f98 task: text_generation precision: fp16 bundle: nemotron-nano-4b.bundle diff --git a/families/locateanything/tests/benchmark/locateanything-3b.yaml b/families/locateanything/tests/benchmark/locateanything-3b.yaml index 2c6f69558d..0a7ef06b87 100644 --- a/families/locateanything/tests/benchmark/locateanything-3b.yaml +++ b/families/locateanything/tests/benchmark/locateanything-3b.yaml @@ -51,6 +51,7 @@ performance: reference: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {workflow: grounded-generation} mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml index f3e0778d24..bdae110537 100644 --- a/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml +++ b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml @@ -6,6 +6,7 @@ model: nllb-200-distilled-600m candidate: family: m2m_100 checkpoint: facebook/nllb-200-distilled-600M + revision: f8d333a098d19b4fd9a8b18f94170487ad3f821d task: text_generation precision: fp16 bundle: nllb-200-distilled-600m.bundle diff --git a/families/marian/tests/benchmark/marian-en-ru.yaml b/families/marian/tests/benchmark/marian-en-ru.yaml index 3ed6debd35..c5e1748028 100644 --- a/families/marian/tests/benchmark/marian-en-ru.yaml +++ b/families/marian/tests/benchmark/marian-en-ru.yaml @@ -6,6 +6,7 @@ model: marian-en-ru candidate: family: marian checkpoint: Helsinki-NLP/opus-mt-en-ru + revision: bb09c99d180016eac6819df3dae68edb1690fdee task: text_generation precision: fp16 bundle: marian-en-ru.bundle diff --git a/families/mistral/tests/benchmark/mistral-7b.yaml b/families/mistral/tests/benchmark/mistral-7b.yaml index b7c486266e..8f29cf0736 100644 --- a/families/mistral/tests/benchmark/mistral-7b.yaml +++ b/families/mistral/tests/benchmark/mistral-7b.yaml @@ -6,6 +6,7 @@ model: mistral-7b candidate: family: mistral checkpoint: mistralai/Mistral-7B-Instruct-v0.1 + revision: ec5deb64f2c6e6fa90c1abf74a91d5c93a9669ca task: text_generation precision: fp16 bundle: mistral-7b.bundle diff --git a/families/mistral/tests/benchmark/riva-translate-4b.yaml b/families/mistral/tests/benchmark/riva-translate-4b.yaml index 1ac4752b33..8b2f8e2559 100644 --- a/families/mistral/tests/benchmark/riva-translate-4b.yaml +++ b/families/mistral/tests/benchmark/riva-translate-4b.yaml @@ -6,6 +6,7 @@ model: riva-translate-4b candidate: family: mistral checkpoint: nvidia/Riva-Translate-4B-Instruct-v1.1 + revision: 1eb944ce9f2717d7be6b7b7c2310f7c58b35d10b task: text_generation precision: fp16 bundle: riva-translate-4b.bundle diff --git a/families/modernbert/tests/benchmark/modernbert-base.yaml b/families/modernbert/tests/benchmark/modernbert-base.yaml index 08cce686c6..1b38bebee2 100644 --- a/families/modernbert/tests/benchmark/modernbert-base.yaml +++ b/families/modernbert/tests/benchmark/modernbert-base.yaml @@ -6,6 +6,7 @@ model: modernbert-base candidate: family: modernbert checkpoint: answerdotai/ModernBERT-base + revision: 8949b909ec900327062f0ebf497f51aef5e6f0c8 task: encoding precision: fp32 bundle: modernbert-base.bundle diff --git a/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml b/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml index e460796826..1c555ea9fd 100644 --- a/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml +++ b/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml @@ -6,6 +6,7 @@ model: all-mpnet-base-v2 candidate: family: mpnet checkpoint: sentence-transformers/all-mpnet-base-v2 + revision: e8c3b32edf5434bc2275fc9bab85f82640a19130 task: encoding precision: fp16 bundle: all-mpnet-base-v2.bundle diff --git a/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml b/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml index 52f4e0aaf5..39de53d6ba 100644 --- a/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml +++ b/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml @@ -6,6 +6,7 @@ model: nemotron-mini-4b candidate: family: nemotron checkpoint: nvidia/Nemotron-Mini-4B-Instruct + revision: 791833e92ebddb0bc2c1007f6d2b6764f886a2ae task: text_generation precision: fp16 bundle: nemotron-mini-4b.bundle diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml index eafdd865cd..7159fbb9cc 100644 --- a/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml @@ -6,6 +6,7 @@ model: nemotron-3.5-asr-streaming-0.6b candidate: family: nemotron_speech_streaming checkpoint: nvidia/nemotron-3.5-asr-streaming-0.6b + revision: ea30d66debe3740a08b573244286791d423d6b3e task: transcription_streaming precision: fp16 bundle: nemotron-3.5-asr-streaming-0.6b.bundle @@ -46,6 +47,7 @@ performance: reference: runner: task-reference adapter: nemo-asr + adapter_options: {transcription_input: manifest, model_loader: prompted-nemo-archive} mode: pytorch-eager reference_backend: nemo_reference precision: fp32 diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml index 4695f1c80d..b956e3d29b 100644 --- a/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml @@ -6,6 +6,7 @@ model: nemotron-speech-streaming-en-0.6b candidate: family: nemotron_speech_streaming checkpoint: nvidia/nemotron-speech-streaming-en-0.6b + revision: ebe59e5a817142986528bbbee5dba8db7b38ed50 task: transcription_streaming precision: fp16 bundle: nemotron-speech-streaming-en-0.6b.bundle @@ -43,6 +44,7 @@ performance: reference: runner: task-reference adapter: nemo-asr + adapter_options: {transcription_input: manifest} mode: pytorch-eager reference_backend: nemo_reference precision: fp32 diff --git a/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml b/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml index 6542bd2612..488e7c5b15 100644 --- a/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml +++ b/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml @@ -47,6 +47,7 @@ performance: reference: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtsmixer} mode: pytorch-eager precision: fp32 output_contract: forecast-parity diff --git a/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml b/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml index 18a49a6f96..828864d1be 100644 --- a/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml +++ b/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml @@ -47,6 +47,7 @@ performance: reference: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtst} mode: pytorch-eager precision: fp32 output_contract: forecast-parity diff --git a/families/patchtst/tests/benchmark/patchtst-granite-official.yaml b/families/patchtst/tests/benchmark/patchtst-granite-official.yaml index 977a29d238..b75284dd56 100644 --- a/families/patchtst/tests/benchmark/patchtst-granite-official.yaml +++ b/families/patchtst/tests/benchmark/patchtst-granite-official.yaml @@ -45,6 +45,7 @@ performance: reference: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtst} mode: pytorch-eager precision: fp32 output_contract: forecast-parity diff --git a/families/phi/tests/benchmark/phi3-mini.yaml b/families/phi/tests/benchmark/phi3-mini.yaml index 8121e59a74..a46f7a20ad 100644 --- a/families/phi/tests/benchmark/phi3-mini.yaml +++ b/families/phi/tests/benchmark/phi3-mini.yaml @@ -6,6 +6,7 @@ model: phi3-mini candidate: family: phi checkpoint: microsoft/Phi-3-mini-4k-instruct + revision: f39ac1d28e925b323eae81227eaba4464caced4e task: text_generation precision: fp16 bundle: phi3-mini.bundle diff --git a/families/phi_moe/tests/benchmark/phi-moe.yaml b/families/phi_moe/tests/benchmark/phi-moe.yaml index 6a9d2a1076..cd4fe380ab 100644 --- a/families/phi_moe/tests/benchmark/phi-moe.yaml +++ b/families/phi_moe/tests/benchmark/phi-moe.yaml @@ -6,6 +6,7 @@ model: phi-moe candidate: family: phi_moe checkpoint: microsoft/Phi-tiny-MoE-instruct + revision: 2fe50e88d0e2a5a132563815686ea0dcc8e252b5 task: text_generation precision: fp16 bundle: phi-moe.bundle diff --git a/families/pixart/tests/benchmark/pixart-sigma-1024.yaml b/families/pixart/tests/benchmark/pixart-sigma-1024.yaml index 1e9b265ac0..c20fc721f1 100644 --- a/families/pixart/tests/benchmark/pixart-sigma-1024.yaml +++ b/families/pixart/tests/benchmark/pixart-sigma-1024.yaml @@ -5,6 +5,7 @@ model: pixart-sigma-1024 candidate: family: pixart checkpoint: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS + revision: e102b3591cc82e97071b8b4cb90d834d0c487207 task: image_generation precision: fp16 bundle: pixart-sigma-1024.bundle diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml index 932354a06b..e3aa7ca591 100644 --- a/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml +++ b/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml @@ -6,6 +6,7 @@ model: qwen3-0.6b-fp16 candidate: family: qwen checkpoint: Qwen/Qwen3-0.6B + revision: c1899de289a04d12100db370d81485cdf75e47ca task: text_generation precision: fp16 bundle: qwen3-0.6b-fp16.bundle diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml index fa220e695b..4f8348834e 100644 --- a/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml +++ b/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml @@ -6,6 +6,7 @@ model: qwen3-0.6b-fp8 candidate: family: qwen checkpoint: Qwen/Qwen3-0.6B + revision: c1899de289a04d12100db370d81485cdf75e47ca task: text_generation precision: fp16 bundle: qwen3-0.6b-fp8-fp16base.bundle diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml index c9a9d2f944..e0d35d76d9 100644 --- a/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml +++ b/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml @@ -6,6 +6,7 @@ model: qwen3-0.6b-topp candidate: family: qwen checkpoint: Qwen/Qwen3-0.6B + revision: c1899de289a04d12100db370d81485cdf75e47ca task: text_generation precision: fp16 bundle: qwen3-0.6b-topp-fp16.bundle diff --git a/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml b/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml index 461a13800e..502b2f4bb6 100644 --- a/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml +++ b/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml @@ -6,6 +6,7 @@ model: qwen3-4b-instruct-2507 candidate: family: qwen checkpoint: Qwen/Qwen3-4B-Instruct-2507 + revision: cdbee75f17c01a7cc42f958dc650907174af0554 task: text_generation precision: fp16 bundle: qwen3-4b-instruct-2507.bundle diff --git a/families/qwen_image/tests/benchmark/qwen-image-2512.yaml b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml index b286f40404..6bf60f4e27 100644 --- a/families/qwen_image/tests/benchmark/qwen-image-2512.yaml +++ b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: qwen-image-2512 -candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-2512, task: image_generation, precision: bf16, bundle: qwen-image-2512.bundle, build: {image_height: 1024, image_width: 1024}} +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-2512, revision: 25468b98e3276ca6700de15c6628e51b7de54a26, task: image_generation, precision: bf16, bundle: qwen-image-2512.bundle, build: {image_height: 1024, image_width: 1024}} reference_environment: {requirements: ../../requirements.txt} accuracy: - - {name: generated-media-parity, benchmark: task_output_parity, operation: generate_image, request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024}, reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}}, gate: {min_psnr: 5.0, min_ssim: 0.1}} + - {name: generated-media-parity, benchmark: task_output_parity, operation: generate_image, request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024}, reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true, always_negative_prompt: true, guidance_parameter: true_cfg_scale}}, gate: {min_psnr: 5.0, min_ssim: 0.1}} performance: - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 10, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml b/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml index b284ae0eeb..f9b14d7a25 100644 --- a/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml +++ b/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml @@ -2,14 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: qwen-image-edit-2511 -candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-Edit-2511, task: image_edit, precision: bf16, bundle: qwen-image-edit-2511.bundle, build: {image_height: 1024, image_width: 1024}} +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-Edit-2511, revision: 6f3ccc0b56e431dc6a0c2b2039706d7d26f22cb9, task: image_edit, precision: bf16, bundle: qwen-image-edit-2511.bundle, build: {image_height: 1024, image_width: 1024}} reference_environment: {requirements: ../../requirements.txt} accuracy: - name: generated-media-parity benchmark: task_output_parity operation: generate_image request: &request {prompt: Turn the image into a watercolor illustration while preserving the main subject., image_path: ../data/test_img.jpeg, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024} - reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true, always_negative_prompt: true, guidance_parameter: true_cfg_scale}} gate: {min_psnr: 5.0, min_ssim: 0.1} performance: - {name: edit-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_image/tests/benchmark/qwen-image.yaml b/families/qwen_image/tests/benchmark/qwen-image.yaml index d3c0354f55..9103502bfc 100644 --- a/families/qwen_image/tests/benchmark/qwen-image.yaml +++ b/families/qwen_image/tests/benchmark/qwen-image.yaml @@ -2,14 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: qwen-image -candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image, task: image_generation, precision: bf16, bundle: qwen-image.bundle, build: {image_height: 1024, image_width: 1024}} +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image, revision: 75e0b4be04f60ec59a75f475837eced720f823b6, task: image_generation, precision: bf16, bundle: qwen-image.bundle, build: {image_height: 1024, image_width: 1024}} reference_environment: {requirements: ../../requirements.txt} accuracy: - name: generated-media-parity benchmark: task_output_parity operation: generate_image request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024} - reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true}} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true, always_negative_prompt: true, guidance_parameter: true_cfg_scale}} gate: {min_psnr: 5.0, min_ssim: 0.1} performance: - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/roberta/tests/benchmark/camembert-base.yaml b/families/roberta/tests/benchmark/camembert-base.yaml index 41c8fbabbf..95f63182ac 100644 --- a/families/roberta/tests/benchmark/camembert-base.yaml +++ b/families/roberta/tests/benchmark/camembert-base.yaml @@ -6,6 +6,7 @@ model: camembert-base candidate: family: roberta checkpoint: almanach/camembert-base + revision: a75967561c78f2aa81cc41045378d3b4ee25af9e task: encoding precision: fp16 bundle: camembert-base.bundle diff --git a/families/roberta/tests/benchmark/roberta-base.yaml b/families/roberta/tests/benchmark/roberta-base.yaml index 54d104417d..98885944de 100644 --- a/families/roberta/tests/benchmark/roberta-base.yaml +++ b/families/roberta/tests/benchmark/roberta-base.yaml @@ -6,6 +6,7 @@ model: roberta-base candidate: family: roberta checkpoint: FacebookAI/roberta-base + revision: e2da8e2f811d1448a5b465c236feacd80ffbac7b task: encoding precision: fp16 bundle: roberta-base.bundle diff --git a/families/roberta/tests/benchmark/roberta-large.yaml b/families/roberta/tests/benchmark/roberta-large.yaml index 439cfffdf4..4cafecb290 100644 --- a/families/roberta/tests/benchmark/roberta-large.yaml +++ b/families/roberta/tests/benchmark/roberta-large.yaml @@ -6,6 +6,7 @@ model: roberta-large candidate: family: roberta checkpoint: FacebookAI/roberta-large + revision: 722cf37b1afa9454edce342e7895e588b6ff1d59 task: encoding precision: fp16 bundle: roberta-large.bundle diff --git a/families/roberta/tests/benchmark/xlm-roberta-base.yaml b/families/roberta/tests/benchmark/xlm-roberta-base.yaml index 92dd3dbc4d..a53a2c68b0 100644 --- a/families/roberta/tests/benchmark/xlm-roberta-base.yaml +++ b/families/roberta/tests/benchmark/xlm-roberta-base.yaml @@ -6,6 +6,7 @@ model: xlm-roberta-base candidate: family: roberta checkpoint: FacebookAI/xlm-roberta-base + revision: e73636d4f797dec63c3081bb6ed5c7b0bb3f2089 task: encoding precision: fp16 bundle: xlm-roberta-base.bundle diff --git a/families/segformer/tests/benchmark/segformer-b0-ade.yaml b/families/segformer/tests/benchmark/segformer-b0-ade.yaml index 37c095207c..60bcd4300c 100644 --- a/families/segformer/tests/benchmark/segformer-b0-ade.yaml +++ b/families/segformer/tests/benchmark/segformer-b0-ade.yaml @@ -40,6 +40,7 @@ performance: reference: runner: task-reference adapter: hf-transformers-vision + adapter_options: {vision_task: semantic-segmentation} mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/starcoder2/tests/benchmark/starcoder2-3b.yaml b/families/starcoder2/tests/benchmark/starcoder2-3b.yaml index 88c6eea1f5..428dc7cbe6 100644 --- a/families/starcoder2/tests/benchmark/starcoder2-3b.yaml +++ b/families/starcoder2/tests/benchmark/starcoder2-3b.yaml @@ -6,6 +6,7 @@ model: starcoder2-3b candidate: family: starcoder2 checkpoint: bigcode/starcoder2-3b + revision: 733247c55e3f73af49ce8e9c7949bf14af205928 task: text_generation precision: fp16 bundle: starcoder2-3b.bundle diff --git a/families/t5/tests/benchmark/t5-small.yaml b/families/t5/tests/benchmark/t5-small.yaml index f97cb56594..98c7f60b8d 100644 --- a/families/t5/tests/benchmark/t5-small.yaml +++ b/families/t5/tests/benchmark/t5-small.yaml @@ -6,6 +6,7 @@ model: t5-small candidate: family: t5 checkpoint: google-t5/t5-small + revision: df1b051c49625cf57a3d0d8d3863ed4d13564fe4 task: text_generation precision: fp16 bundle: t5-small.bundle diff --git a/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml b/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml index 95176a8f27..d0e00fffa0 100644 --- a/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml +++ b/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml @@ -43,6 +43,7 @@ performance: reference: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: timesfm} mode: pytorch-eager precision: fp32 output_contract: forecast-parity diff --git a/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml index 837a24e586..08a5f80211 100644 --- a/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml +++ b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml @@ -6,6 +6,7 @@ model: densenet121-ra-in1k candidate: family: timm_densenet checkpoint: timm/densenet121.ra_in1k + revision: 92007b6200e0b4a4fe68cb4e3947022a928aaaae task: image_to_class_scores precision: fp16 bundle: densenet121-ra-in1k.bundle diff --git a/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml index 54d580fb30..06514d194d 100644 --- a/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml +++ b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml @@ -6,6 +6,7 @@ model: efficientnet-b0-ra-in1k candidate: family: timm_efficientnet checkpoint: timm/efficientnet_b0.ra_in1k + revision: 1b5383e5f79cc0f7fc067e372f8f26a5fa73f26a task: image_to_class_scores precision: fp16 bundle: efficientnet-b0-ra-in1k.bundle diff --git a/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml index 08011445ea..8fc6b730d4 100644 --- a/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml +++ b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml @@ -6,6 +6,7 @@ model: inception-v3-tv-in1k candidate: family: timm_inception checkpoint: timm/inception_v3.tv_in1k + revision: 393d84cc85c467d8fbc0dc81a65c04e87a32572c task: image_to_class_scores precision: fp16 bundle: inception-v3-tv-in1k.bundle diff --git a/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml b/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml index 350285a356..830276980b 100644 --- a/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml +++ b/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml @@ -6,6 +6,7 @@ model: mnasnet-100-rmsp-in1k candidate: family: timm_mnasnet checkpoint: timm/mnasnet_100.rmsp_in1k + revision: a30af72360c7a4871b0156cc11d7b767208273d7 task: classification precision: fp16 bundle: mnasnet-100-rmsp-in1k.bundle diff --git a/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml index 79b729f77a..82fe4490c8 100644 --- a/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml +++ b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml @@ -6,6 +6,7 @@ model: mobilenetv2-100-ra-in1k candidate: family: timm_mobilenetv2 checkpoint: timm/mobilenetv2_100.ra_in1k + revision: 5afa12513b048c79b9147a5d210e9bdc50035481 task: image_to_class_scores precision: fp16 bundle: mobilenetv2-100-ra-in1k.bundle diff --git a/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml index ae044e5f85..674895349a 100644 --- a/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml +++ b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml @@ -6,6 +6,7 @@ model: mobilenetv3-large-100-ra-in1k candidate: family: timm_mobilenetv3 checkpoint: timm/mobilenetv3_large_100.ra_in1k + revision: 96f46a1c52932f27492dff66c72378eb99b443a7 task: image_to_class_scores precision: fp16 bundle: mobilenetv3-large-100-ra-in1k.bundle diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml index 893a2a9863..58187d26af 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml @@ -6,6 +6,7 @@ model: mobilenetv4-conv-large candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_large.e500_r256_in1k + revision: 033de0bd74b5261f7339c5de242a2e4785808aa3 task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-large.bundle diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml index 45065bde30..77426a40a9 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml @@ -6,6 +6,7 @@ model: mobilenetv4-conv-medium candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_medium.e500_r256_in1k + revision: ad66898c045c1b5223ea3f2c0830b74cf2e75bac task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-medium.bundle diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml index 4b2a6e84bc..0865e90685 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml @@ -6,6 +6,7 @@ model: mobilenetv4-conv-small-050 candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_small_050.e3000_r224_in1k + revision: 0ca27fbda87631d17b5a3d348af8e6b150dd9282 task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-small-050.bundle diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml index 10e9af4330..78ac621073 100644 --- a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml @@ -6,6 +6,7 @@ model: mobilenetv4-conv-small candidate: family: timm_mobilenetv4 checkpoint: timm/mobilenetv4_conv_small.e2400_r224_in1k + revision: 331fb803779522b685cf942e15f914fb6741c1eb task: image_to_class_scores precision: fp16 bundle: mobilenetv4-conv-small.bundle diff --git a/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml index b4a6d72b32..daa9f67758 100644 --- a/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml +++ b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml @@ -6,6 +6,7 @@ model: resnet50-a1-in1k candidate: family: timm_resnet checkpoint: timm/resnet50.a1_in1k + revision: 767268603ca0cb0bfe326fa87277f19c419566ef task: image_to_class_scores precision: fp16 bundle: resnet50-a1-in1k.bundle diff --git a/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml b/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml index fd7368f3bf..a9caf433e5 100644 --- a/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml +++ b/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml @@ -6,6 +6,7 @@ model: vgg16-tv-in1k candidate: family: timm_vgg checkpoint: timm/vgg16.tv_in1k + revision: b8d8aa2dd860af9233c8c67385a8097fd6c35d3f task: classification precision: fp16 bundle: vgg16-tv-in1k.bundle diff --git a/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml index 36bf5a9db5..ec37c0bc34 100644 --- a/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml +++ b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml @@ -6,6 +6,7 @@ model: timm-vit-base-p16-224-augreg-in21k-ft-in1k candidate: family: timm_vit checkpoint: timm/vit_base_patch16_224.augreg_in21k_ft_in1k + revision: 2ec9fb3d7bb664aac471ac44582c94d18de33780 task: image_to_class_scores precision: fp16 bundle: timm-vit-base-p16-224-augreg-in21k-ft-in1k.bundle diff --git a/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml b/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml index 2b406e15de..e17ac3f375 100644 --- a/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml +++ b/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml @@ -9,7 +9,7 @@ accuracy: benchmark: task_output_parity operation: generate_image request: &request {prompt: Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage, media_type: video, seed: 42, num_steps: 50, guidance_scale: 5.0, height: 704, width: 1280, video_num_frames: 121} - reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {model_id: Wan-AI/Wan2.2-TI2V-5B-Diffusers, model_revision: b8fff7315c768468a5333511427288870b2e9635}} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {model_id: Wan-AI/Wan2.2-TI2V-5B-Diffusers, model_revision: b8fff7315c768468a5333511427288870b2e9635, vae_class: AutoencoderKLWan, vae_precision: fp32}} gate: {min_psnr: 5.0, min_ssim: 0.1} performance: - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml b/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml index a8f969e0b1..ab99b990fb 100644 --- a/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml +++ b/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: wan21-t2v-1.3b -candidate: {family: wan_t2v, checkpoint: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, task: image_generation, precision: fp16, bundle: wan21-t2v-1.3b.bundle, build: {context_parallel_size: 1, fp32_layers: [24], image_height: 480, image_width: 832, video_num_frames: 17}} +candidate: {family: wan_t2v, checkpoint: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, revision: 0fad780a534b6463e45facd96134c9f345acfa5b, task: image_generation, precision: fp16, bundle: wan21-t2v-1.3b.bundle, build: {context_parallel_size: 1, fp32_layers: [24], image_height: 480, image_width: 832, video_num_frames: 17}} reference_environment: {requirements: ../../requirements.txt} accuracy: - name: generated-media-parity diff --git a/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml b/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml index 4bd54f5cd2..9caf9e4c79 100644 --- a/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml +++ b/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml @@ -6,6 +6,7 @@ model: whisper-large-v3-turbo candidate: family: whisper checkpoint: openai/whisper-large-v3-turbo + revision: 41f01f3fe87f28c78e2fbf8b568835947dd65ed9 task: transcription precision: fp32 bundle: whisper-large-v3-turbo.bundle diff --git a/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml b/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml index f0306c82eb..4900ee70bf 100644 --- a/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml +++ b/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml @@ -6,6 +6,7 @@ model: whisper-tiny-fp16 candidate: family: whisper checkpoint: openai/whisper-tiny + revision: 169d4a4341b33bc18d8881c4b69c2e104e1cc0af task: transcription precision: fp16 bundle: whisper-tiny-fp16.bundle diff --git a/families/xglm/tests/benchmark/xglm-564m.yaml b/families/xglm/tests/benchmark/xglm-564m.yaml index a179be3ef7..3bb2f771ba 100644 --- a/families/xglm/tests/benchmark/xglm-564m.yaml +++ b/families/xglm/tests/benchmark/xglm-564m.yaml @@ -6,6 +6,7 @@ model: xglm-564m candidate: family: xglm checkpoint: facebook/xglm-564M + revision: f3059f01b98ccc877c673149e0178c0e957660f9 task: text_generation precision: fp16 bundle: xglm-564m.bundle diff --git a/families/xlnet/tests/benchmark/xlnet-base.yaml b/families/xlnet/tests/benchmark/xlnet-base.yaml index 441f07473d..887f702194 100644 --- a/families/xlnet/tests/benchmark/xlnet-base.yaml +++ b/families/xlnet/tests/benchmark/xlnet-base.yaml @@ -6,6 +6,7 @@ model: xlnet-base candidate: family: xlnet checkpoint: xlnet/xlnet-base-cased + revision: ceaa69c7bc5e512b5007106a7ccbb7daf24b2c79 task: encoding precision: fp16 bundle: xlnet-base.bundle diff --git a/families/z_image/tests/benchmark/z-image-turbo.yaml b/families/z_image/tests/benchmark/z-image-turbo.yaml index d2a4d7a217..5cb95d6d60 100644 --- a/families/z_image/tests/benchmark/z-image-turbo.yaml +++ b/families/z_image/tests/benchmark/z-image-turbo.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 model: z-image-turbo -candidate: {family: z_image, checkpoint: Tongyi-MAI/Z-Image-Turbo, task: image_generation, precision: fp16, bundle: z-image-turbo.bundle, build: {fp32_layers: [2, 3, 4, 7, 8], image_height: 1024, image_width: 1024}} +candidate: {family: z_image, checkpoint: Tongyi-MAI/Z-Image-Turbo, revision: f332072aa78be7aecdf3ee76d5c247082da564a6, task: image_generation, precision: fp16, bundle: z-image-turbo.bundle, build: {fp32_layers: [2, 3, 4, 7, 8], image_height: 1024, image_width: 1024}} reference_environment: {requirements: ../../requirements.txt} accuracy: - name: generated-media-parity diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py index 96bc3a38a7..e88d5dd2fe 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/tools/benchmark_qualification/accuracy.py @@ -1437,6 +1437,15 @@ def _candidate_outputs( if not isinstance(cells, list) or len(cells) != len(requests): raise QualificationError("trtmc-bench returned an invalid Accuracy result set") outputs = [] + candidate_root = candidate_output.resolve() + + def resolve_artifact(artifact_root: Path, value: str) -> str: + path = Path(value) + resolved = path.resolve() if path.is_absolute() else (artifact_root / path).resolve() + if not resolved.is_relative_to(artifact_root): + raise QualificationError("trtmc-bench returned an unsafe artifact path") + return str(resolved) + for cell in cells: if not isinstance(cell, Mapping) or cell.get("status") != "completed": raise QualificationError("trtmc-bench reported a failed Accuracy request") @@ -1447,21 +1456,14 @@ def _candidate_outputs( artifact_dir = cell.get("artifact_dir") if isinstance(artifact_dir, str) and artifact_dir: artifact_root = (candidate_output / artifact_dir).resolve() - if not artifact_root.is_relative_to(candidate_output.resolve()): + if not artifact_root.is_relative_to(candidate_root): raise QualificationError("trtmc-bench returned an unsafe artifact directory") for name, value in tuple(resolved_summary.items()): if name.endswith("_artifact") and isinstance(value, str): - path = Path(value) - resolved_summary[name] = str( - path if path.is_absolute() else (artifact_root / path).resolve() - ) + resolved_summary[name] = resolve_artifact(artifact_root, value) elif name.endswith("_artifacts") and isinstance(value, list): resolved_summary[name] = [ - str( - Path(item) - if Path(item).is_absolute() - else (artifact_root / item).resolve() - ) + resolve_artifact(artifact_root, item) for item in value if isinstance(item, str) ] diff --git a/tools/benchmark_qualification/catalog.py b/tools/benchmark_qualification/catalog.py index c9546eae76..17caf44f7b 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/tools/benchmark_qualification/catalog.py @@ -186,6 +186,12 @@ def _candidate(raw: Any, family: str, path: Path) -> dict[str, Any]: raise QualificationError( f"{path}: trusted remote code requires an immutable 40-character revision" ) + if model_directory is None and ( + not isinstance(revision, str) or re.fullmatch(r"[0-9a-fA-F]{40}", revision) is None + ): + raise QualificationError( + f"{path}: remote checkpoint requires an immutable 40-character revision" + ) build = value.get("build", {}) if not isinstance(build, Mapping): raise QualificationError(f"{path}: candidate.build must be an object") diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 2feac9b14a..f5d48edaec 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -80,7 +80,8 @@ def test_qualification_yaml_preserves_merge_keys(tmp_path: Path) -> None: profile.write_text( "schema_version: trtmc.qualification/v1\n" "model: example-model\n" - "candidate: {family: example, checkpoint: example/model, task: example_task, " + "candidate: {family: example, checkpoint: example/model, revision: " + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, task: example_task, " "precision: fp32}\n" "reference: &reference {runner: hf-transformers, mode: hf-eager}\n" "accuracy: []\n" @@ -185,6 +186,7 @@ def test_l0_configs_outside_the_benchmark_folder_are_not_discovered( "candidate:\n" " family: example\n" " checkpoint: example/model\n" + " revision: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" " task: text_generation\n" " precision: fp16\n" "accuracy: []\n" @@ -221,6 +223,31 @@ def test_trusted_remote_code_requires_an_immutable_revision( discover(tmp_path) +@pytest.mark.parametrize("revision", [None, "main", "abc1234"]) +def test_remote_checkpoint_requires_an_immutable_revision( + tmp_path: Path, revision: str | None +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + revision_line = "" if revision is None else f" revision: {revision}\n" + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + f"{revision_line}" + " task: text_generation\n" + " precision: fp16\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="remote checkpoint requires an immutable"): + discover(tmp_path) + + @pytest.mark.parametrize("model_directory", ["/tmp/model", "../model"]) def test_candidate_model_directory_must_stay_in_the_family_environment( tmp_path: Path, model_directory: str @@ -256,6 +283,7 @@ def test_one_model_file_owns_multiple_cases_without_testcase_indirection( "candidate:\n" " family: example\n" " checkpoint: example/model\n" + " revision: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" " task: text_generation\n" " precision: fp16\n" "accuracy:\n" @@ -288,6 +316,7 @@ def test_family_environment_hook_is_discovered_with_the_model_profiles(tmp_path: "candidate:\n" " family: example\n" " checkpoint: example/model\n" + " revision: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" " task: text_generation\n" " precision: fp16\n" "accuracy:\n" @@ -2207,6 +2236,83 @@ def test_candidate_descriptor_uses_family_environment_model_directory( ) +@pytest.mark.parametrize("artifact_path", ["../outside.npy", "absolute"]) +def test_accuracy_rejects_candidate_artifacts_outside_the_cell_directory( + tmp_path: Path, monkeypatch, artifact_path: str +) -> None: + case = _example_case(tmp_path) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + outside = tmp_path / "outside.npy" + reported_path = str(outside.resolve()) if artifact_path == "absolute" else artifact_path + + monkeypatch.setattr( + qualification_accuracy, + "require_candidate", + lambda _context: (tmp_path / "worker", tmp_path / "runtime"), + ) + monkeypatch.setattr( + qualification_accuracy, + "write_model_descriptor", + lambda *_args, **_kwargs: tmp_path / "model.json", + ) + monkeypatch.setattr( + qualification_accuracy, + "prepare_bundle", + lambda *_args, **_kwargs: tmp_path / "model.bundle", + ) + monkeypatch.setattr( + qualification_accuracy, + "benchmark_executable", + lambda *_args, **_kwargs: tmp_path / "trtmc-bench", + ) + + def candidate(command, *_args, **_kwargs): + candidate_output = Path(command[command.index("--output") + 1]) + candidate_output.mkdir(parents=True) + (candidate_output / "result.json").write_text( + json.dumps( + { + "cells": [ + { + "status": "completed", + "artifact_dir": "artifacts/case", + "output_summary": {"mask_artifact": reported_path}, + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(qualification_accuracy, "run_command", candidate) + + output = tmp_path / "qualification" + output.mkdir() + with pytest.raises(QualificationError, match="unsafe artifact path"): + qualification_accuracy._candidate_outputs( + case, + context, + output, + "segment", + [{"request": {"image_path": "/data/image.jpg"}}], + ) + + @pytest.mark.parametrize( ("task", "public_request", "expected_inputs", "expected_controls"), [ From c2446d20f3f454918ce7095a76119e53ee208401 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 11:40:25 +0000 Subject: [PATCH 097/100] refactor(benchmark): separate qualification runtime Keep trtmc-bench focused on the installed user workflow while moving repository-only Accuracy and Performance orchestration under qualification_tests. Run candidates through the installed benchmark, share one reference protocol, and keep model-specific reference behavior and tests under the owning family. Signed-off-by: chaofengw --- apps/benchmark/performance/README.md | 376 -- .../performance/baselines/elf_reference.py | 392 -- .../baselines/reference_support.py | 54 - .../performance/baselines/task_reference.py | 3240 ----------------- .../tests/test_reporting_and_cache.py | 48 - .../canary/tests/benchmark/canary-1b-v2.yaml | 2 +- families/canary/tests/benchmark/reference.py | 66 +- .../tests/benchmark/deepseek-ocr.yaml | 3 +- .../deepseek_ocr/tests/benchmark/reference.py | 79 +- .../tests/test_benchmark_reference.py | 19 + .../detr/tests/benchmark/detr-resnet-50.yaml | 3 +- families/detr/tests/benchmark/reference.py | 72 +- ...inov3-convnext-tiny-pretrain-lvd1689m.yaml | 3 +- .../dinov3-vits16-pretrain-lvd1689m.yaml | 3 +- families/dinov3/tests/benchmark/reference.py | 57 +- .../tests/benchmark/glmasr-nano-fp16.yaml | 2 +- families/glmasr/tests/benchmark/reference.py | 95 +- .../tests/benchmark/lance-3b-x2t-image.yaml | 2 +- families/lance/tests/benchmark/reference.py | 98 + .../tests/benchmark/upstream_reference.py | 0 .../lance/tests/test_benchmark_reference.py | 43 + .../tests/test_benchmark_reference.py | 19 + .../tests/benchmark/locateanything-3b.yaml | 3 +- .../tests/benchmark/reference.py | 54 +- .../tests/test_benchmark_reference.py | 19 + .../tests/benchmark/magpie-tts-357m.yaml | 2 +- .../magpie_tts/tests/benchmark/reference.py | 120 + .../nemotron-3.5-asr-streaming-0.6b.yaml | 3 +- .../nemotron-speech-streaming-en-0.6b.yaml | 3 +- .../tests/benchmark/reference.py | 108 +- .../tests/benchmark/personaplex-7b.yaml | 2 +- .../personaplex/tests/benchmark/reference.py | 130 + .../qwen3-omni-30b-a3b-instruct.yaml | 2 +- .../qwen3_omni/tests/benchmark/reference.py | 111 + .../tests/test_benchmark_reference.py | 24 + families/sana_wm/tests/benchmark/reference.py | 199 + .../benchmark/sana-wm-bidirectional.yaml | 2 +- .../tests/benchmark/upstream_reference.py | 0 .../sana_wm/tests/test_benchmark_reference.py | 662 ++++ .../segformer/tests/benchmark/reference.py | 61 +- .../tests/benchmark/segformer-b0-ade.yaml | 3 +- families/whisper/tests/benchmark/reference.py | 2 +- .../skills/optimize-model-precision/SKILL.md | 2 +- .../skills/profile-model/SKILL.md | 6 +- pyproject.toml | 3 +- .../__init__.py | 2 +- .../benchmark_qualification/__init__.py | 4 + .../benchmark_qualification/accuracy.py | 107 +- .../audio_generation_model_performance.yaml | 0 ...audio_generation_pipeline_performance.yaml | 0 .../benchmarks/beans_image_feature_knn.yaml | 0 .../benchmarks/coco2017_object_detection.yaml | 0 .../benchmarks/embedding_performance.yaml | 0 .../benchmarks/encoder_performance.yaml | 0 .../benchmarks/etth1_forecast_parity.yaml | 0 .../benchmarks/flores200_en_fr_seq2seq.yaml | 0 .../flores200_en_fr_translation.yaml | 0 .../benchmarks/humaneval_continuation.yaml | 0 .../image_classification_performance.yaml | 0 .../image_features_performance.yaml | 0 .../benchmarks/imagenette_classification.yaml | 0 .../imagenette_localization_parity.yaml | 0 ...agenette_prompted_segmentation_parity.yaml | 0 .../imagenette_segmentation_parity.yaml | 0 ...genette_text_prompted_instance_parity.yaml | 0 .../imagenette_vision_language_parity.yaml | 0 .../benchmarks/librispeech_transcription.yaml | 0 .../media_generation_performance.yaml | 0 .../benchmarks/metric_geometry_parity.yaml | 0 .../metric_geometry_performance.yaml | 0 .../benchmarks/mmlu_continuation.yaml | 0 .../newstest2019_en_ru_translation.yaml | 0 .../object_detection_performance.yaml | 0 .../benchmarks/ocrbench_v2_parity.yaml | 0 .../benchmarks/reranking_performance.yaml | 0 .../benchmarks/robot_action_parity.yaml | 0 .../benchmarks/robot_action_performance.yaml | 0 .../semantic_segmentation_performance.yaml | 0 .../benchmarks/speech_nemo_performance.yaml | 0 .../speech_transformer_performance.yaml | 0 .../benchmarks/stereo_disparity_parity.yaml | 0 .../stereo_disparity_performance.yaml | 0 .../stsbenchmark_embedding_parity.yaml | 0 .../stsbenchmark_reranking_parity.yaml | 0 .../benchmarks/task_output_parity.yaml | 0 .../text_generation_performance.yaml | 0 ...ext_prompted_segmentation_performance.yaml | 0 .../benchmarks/time_series_performance.yaml | 0 .../vision_language_performance.yaml | 0 .../vision_language_pipeline_performance.yaml | 0 .../benchmarks/wikitext103_continuation.yaml | 0 .../benchmarks/wmt14_en_de_translation.yaml | 0 .../benchmark_qualification/catalog.py | 2 +- .../benchmark_qualification/datasets.py | 0 .../performance/README.md | 95 + .../performance/__init__.py | 9 + .../performance/__main__.py | 8 + .../config}/environments/auto-thor.yaml | 4 +- .../config}/environments/gb300.yaml | 4 +- .../config}/environments/l4t-thor.yaml | 4 +- .../performance/config}/release.yaml | 45 +- .../performance/config}/requirements.txt | 0 .../performance/matrix.py | 392 +- .../performance/qualification.py | 158 +- .../performance/reference_harness.py | 233 ++ .../performance/reference_protocol.py | 89 + .../performance/references/__init__.py | 0 .../references}/audio_reference.py | 0 .../references/generic_reference.py | 1685 +++++++++ .../references}/hf_transformers.py | 0 .../references}/timing_contracts.py | 0 .../performance/reporting.py | 203 ++ .../performance/runner.py | 65 + .../performance/stability.py | 102 + .../performance/tests/__init__.py | 4 + .../test_diffusers_semantic_reference.py | 30 +- .../tests/test_family_performance.py | 25 +- .../tests/test_offline_speech_contract.py | 2 +- .../performance}/tests/test_perf_matrix.py | 1398 +------ .../tests/test_reference_protocol.py | 67 + .../performance/tests/test_reporting.py | 56 + .../tests/test_structured_output_contracts.py | 2 +- .../tests/test_timing_contracts.py | 37 +- .../performance/types.py | 47 + .../references/hf_encoder.py | 0 .../references/hf_text_generation.py | 0 .../references/speech_io.py | 0 .../references/timm_image_classification.py | 0 .../benchmark_qualification/runtime.py | 20 +- tools/model_benchmark.py | 8 +- tools/test_impact.py | 3 +- tools/tests/test_architecture.py | 149 +- tools/tests/test_e2e_evidence.py | 2 +- tools/tests/test_family_impact.py | 2 +- tools/tests/test_model_benchmark.py | 124 +- tools/tests/test_new_ci.py | 2 +- website/docs/reference/benchmarking.md | 33 +- .../advanced/validation-and-benchmarking.md | 12 +- 138 files changed, 5148 insertions(+), 6283 deletions(-) delete mode 100644 apps/benchmark/performance/README.md delete mode 100644 apps/benchmark/performance/baselines/elf_reference.py delete mode 100644 apps/benchmark/performance/baselines/reference_support.py delete mode 100644 apps/benchmark/performance/baselines/task_reference.py create mode 100644 families/deepseek_ocr/tests/test_benchmark_reference.py create mode 100644 families/lance/tests/benchmark/reference.py rename apps/benchmark/performance/baselines/lance_reference.py => families/lance/tests/benchmark/upstream_reference.py (100%) create mode 100644 families/lance/tests/test_benchmark_reference.py create mode 100644 families/lerobot_act/tests/test_benchmark_reference.py create mode 100644 families/locateanything/tests/test_benchmark_reference.py create mode 100644 families/magpie_tts/tests/benchmark/reference.py create mode 100644 families/personaplex/tests/benchmark/reference.py create mode 100644 families/qwen3_omni/tests/benchmark/reference.py create mode 100644 families/qwen3_omni/tests/test_benchmark_reference.py create mode 100644 families/sana_wm/tests/benchmark/reference.py rename apps/benchmark/performance/baselines/sana_wm_reference.py => families/sana_wm/tests/benchmark/upstream_reference.py (100%) create mode 100644 families/sana_wm/tests/test_benchmark_reference.py rename {tools/benchmark_qualification => qualification_tests}/__init__.py (65%) create mode 100644 qualification_tests/benchmark_qualification/__init__.py rename {tools => qualification_tests}/benchmark_qualification/accuracy.py (98%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/coco2017_object_detection.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/embedding_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/encoder_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/humaneval_continuation.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/image_classification_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/image_features_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/imagenette_classification.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/librispeech_transcription.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/media_generation_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/metric_geometry_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/metric_geometry_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/mmlu_continuation.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/object_detection_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/reranking_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/robot_action_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/robot_action_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/speech_nemo_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/speech_transformer_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/task_output_parity.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/text_generation_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/time_series_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/vision_language_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/wikitext103_continuation.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml (100%) rename {tools => qualification_tests}/benchmark_qualification/catalog.py (98%) rename {tools => qualification_tests}/benchmark_qualification/datasets.py (100%) create mode 100644 qualification_tests/benchmark_qualification/performance/README.md create mode 100644 qualification_tests/benchmark_qualification/performance/__init__.py create mode 100644 qualification_tests/benchmark_qualification/performance/__main__.py rename {apps/benchmark/performance => qualification_tests/benchmark_qualification/performance/config}/environments/auto-thor.yaml (82%) rename {apps/benchmark/performance => qualification_tests/benchmark_qualification/performance/config}/environments/gb300.yaml (81%) rename {apps/benchmark/performance => qualification_tests/benchmark_qualification/performance/config}/environments/l4t-thor.yaml (82%) rename {apps/benchmark/performance => qualification_tests/benchmark_qualification/performance/config}/release.yaml (97%) rename {apps/benchmark/performance => qualification_tests/benchmark_qualification/performance/config}/requirements.txt (100%) rename tools/perf_matrix.py => qualification_tests/benchmark_qualification/performance/matrix.py (90%) rename tools/benchmark_qualification/performance.py => qualification_tests/benchmark_qualification/performance/qualification.py (62%) create mode 100644 qualification_tests/benchmark_qualification/performance/reference_harness.py create mode 100644 qualification_tests/benchmark_qualification/performance/reference_protocol.py create mode 100644 qualification_tests/benchmark_qualification/performance/references/__init__.py rename {apps/benchmark/performance/baselines => qualification_tests/benchmark_qualification/performance/references}/audio_reference.py (100%) create mode 100644 qualification_tests/benchmark_qualification/performance/references/generic_reference.py rename {apps/benchmark/performance/baselines => qualification_tests/benchmark_qualification/performance/references}/hf_transformers.py (100%) rename {apps/benchmark/performance/baselines => qualification_tests/benchmark_qualification/performance/references}/timing_contracts.py (100%) create mode 100644 qualification_tests/benchmark_qualification/performance/reporting.py create mode 100644 qualification_tests/benchmark_qualification/performance/runner.py create mode 100644 qualification_tests/benchmark_qualification/performance/stability.py create mode 100644 qualification_tests/benchmark_qualification/performance/tests/__init__.py rename {apps/benchmark/trtmc_benchmark => qualification_tests/benchmark_qualification/performance}/tests/test_diffusers_semantic_reference.py (92%) rename {apps/benchmark/trtmc_benchmark => qualification_tests/benchmark_qualification/performance}/tests/test_family_performance.py (97%) rename {apps/benchmark/trtmc_benchmark => qualification_tests/benchmark_qualification/performance}/tests/test_offline_speech_contract.py (99%) rename {apps/benchmark/trtmc_benchmark => qualification_tests/benchmark_qualification/performance}/tests/test_perf_matrix.py (57%) create mode 100644 qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py create mode 100644 qualification_tests/benchmark_qualification/performance/tests/test_reporting.py rename {apps/benchmark/trtmc_benchmark => qualification_tests/benchmark_qualification/performance}/tests/test_structured_output_contracts.py (99%) rename {apps/benchmark/trtmc_benchmark => qualification_tests/benchmark_qualification/performance}/tests/test_timing_contracts.py (81%) create mode 100644 qualification_tests/benchmark_qualification/performance/types.py rename {tools => qualification_tests}/benchmark_qualification/references/hf_encoder.py (100%) rename {tools => qualification_tests}/benchmark_qualification/references/hf_text_generation.py (100%) rename {tools => qualification_tests}/benchmark_qualification/references/speech_io.py (100%) rename {tools => qualification_tests}/benchmark_qualification/references/timm_image_classification.py (100%) rename {tools => qualification_tests}/benchmark_qualification/runtime.py (94%) diff --git a/apps/benchmark/performance/README.md b/apps/benchmark/performance/README.md deleted file mode 100644 index 458ab45ebd..0000000000 --- a/apps/benchmark/performance/README.md +++ /dev/null @@ -1,376 +0,0 @@ - - -# Performance matrix - -The performance matrix is an application above Model Connect: - -~~~text -performance matrix -> trtmc-bench -> public build and Task APIs -performance matrix -> reference runner -~~~ - -Neither the native core nor any family imports benchmark code. The candidate -worker selects the bundle's runtime mode before execution: migrated bundles use -the public header-only Task SDK and C ABI, while existing bundles use -load_task(bundle, runtime_root). Both call the family's implementation directly; -a failed SDK call is never retried through an old interface. - -## Task API benchmark - -Use the packaged CLI for a single model or the checked-in multi-model example: - -```bash -trtmc-bench list models -trtmc-bench run --model distilgpt2 --runtime-root /opt/trtmc/lib -o results/distilgpt2 -trtmc-bench run apps/benchmark/example.yaml -o results/example -``` - -Without another workload, `--model` uses that model's E2E testcase. To benchmark -user input, pass public Task request JSON or JSONL. Each JSONL row may be a raw -request or `{"name": "case-name", "request": {...}}`; relative `*_path` values -are resolved from the data file. - -```bash -trtmc-bench run --model gpt2-125m --data requests.jsonl \ - --runtime-root /opt/trtmc/lib -o results/gpt2-data -``` - -Missing bundles are built through the public build command and cached. Pass -`--no-build` when every selected bundle must already exist. - -Managed cache reuse requires a matching benchmark receipt for the resolved -immutable Hugging Face snapshot, manifest, build arguments, builder sources, -and build environment. Changed or missing identities rebuild the bundle; -`--no-build` reports an error instead. Arbitrary local checkpoint directories -are mutable and are rebuilt rather than assumed unchanged. An explicitly -supplied bundle outside the managed cache remains the caller's provenance -responsibility. `--rebuild` forces a fresh managed build. - -### Select another Task from the same bundle - -A family-owned testcase can select a secondary Task without changing the -manifest's primary `task` or rebuilding the bundle: - -```json -{ - "name": "token-features", - "selected_task": "text_to_token_features", - "inputs": {"token_ids": [7, 9]}, - "config": {} -} -``` - -The existing `--case token-features` option selects this workload. Alternatively, -`--task text_to_token_features` selects the interface for the chosen testcase; -its inputs must match that interface. The operation defaults from the selected -Task. An explicit incompatible operation is an error, not a request to try -another interface. `selected_task` is call selection, not family Config. - -The worker checks that the loaded model actually binds the requested Task. -Bundle identity and managed-cache checks still use the manifest's primary Task. -Results, reproduction records and history comparison retain the selected Task, -so token features and pooled features cannot become the same performance series. -Performance references receive the selection separately from the unchanged -manifest. Their input and output support must match the chosen contract; the -text-only reference loaders reject token IDs instead of replacing them with -empty text. This selection path requires a family migrated to the Task SDK. - -The `head_scores` operation uses `head-scores-shape` for performance comparison: -both outputs must contain every finite score and agree on tensor shape, score -kind, pooling and normalization. It does not compare score accuracy or turn -logits into embeddings. Numerical acceptance remains in the owning family's -correctness tests; a matching reference implementation is still required. - -The `geometry`, `predict_structure` and `refine_pose` operations similarly use -`metric-geometry-shape`, `molecular-structure-shape` and `pose-refinement-shape`. -These contracts check complete output fields and artifact lengths, semantic -layouts, confidence presence and pose callback metadata. Their receipts mark -`numerical_parity_checked: false`: predicted values and opaque serialization -lengths need not be identical across implementations. Existing family numerical -thresholds remain authoritative. - -`offline_speech_dialogue` uses `offline-speech-shape`: the complete candidate -event PCM and reference Float32 WAV must agree on input/output counts and audio -formats. Text is retained per epoch, with final text replacing partial updates -instead of duplicating them. The comparison records both texts but does not -judge text or audio accuracy; `text_parity_checked` and -`numerical_parity_checked` are false. It cannot qualify live or tool dialogue. - -### Family-owned performance declarations - -The canonical release suite also reads optional -`families//tests/performance.yaml` files using the same -`trtmc.perf-suite/v2` format. Each file may name only that family's manifests and -testcases. An owned entry with the same ID and family replaces a central entry -in place; other new IDs are appended. Replacement uses the complete owned -declaration and its own defaults, not a partial merge with the old entry. Retain -the original workload, measurement and comparison criteria explicitly when -migrating a reference. Already-expanded additional profiles require their own -exact-ID replacements. Cross-family ID conflicts, repeated IDs within a family -file and family exclusions remain errors. A standalone user suite does not -implicitly include other suites. Check, prepare, run and resume use the same -composition. -An accepted owned declaration activates only its explicitly named model from -central exclusions for validation; it does not establish qualification. - -Use an existing reference adapter when it implements the actual workload. -Otherwise declare a Python script relative to the owning family directory: - -Reference adapters that need prepared repositories or model directories declare -their inputs on the suite entry instead of relying on a shared adapter registry. -Each option names a field from the selected performance environment; optional -`required` paths are checked below that directory before the reference starts: - -```yaml -baseline: - runner: task-reference - adapter: upstream-example - reference_inputs: - reference_repo: - environment: example_repo - required: [inference.py] -``` - -The environment supplies `references.example_repo`. A family qualification -profile may instead put `reference_repo` directly under -`reference_environment.paths`; the qualification runner forwards all such -family-declared paths as adapter options without knowing their meaning. - -```yaml -schema_version: trtmc.perf-suite/v2 -name: example-performance -defaults: - measurement: {warmup: 3, iterations: 10} -entries: - - id: example.head_scores - family: example - model: example-model - operation: head_scores - workload: {testcase: example-head-scores} - baseline: - runner: task-reference - script: tests/performance_reference.py - mode: pytorch-eager - timing_scope: task-model-call-wall - input_preparation_included: false - asset_loading_included: false -``` - -This is an illustrative declaration, not an existing qualified model. `script` -and `adapter` are mutually exclusive. The script must be an existing `.py` file -inside its family, without symlinks or parent-directory traversal. It runs through -the existing subprocess mechanism; the core does not import or execute it. - -The script receives the existing model/revision, family, operation, manifest, -request JSON, selected Task, adapter-options JSON, precision, mode, padding, -warmup, iterations, case name, output path and timing-contract JSON arguments. -When no mode is declared, a custom script uses the neutral label `reference`, -not an implied `torch.compile` claim. Supplied token IDs stay in request JSON; -the script must execute the requested operand and Task, not decode/re-tokenize or -substitute a different operation. Preparation needed by the candidate must happen -explicitly before the candidate runs; a reference is not a preparation hook. - -Return `trtmc.perf-baseline/v1` JSON with `status: completed`, matching model, -family, operation, case name, selected Task, precision and mode. Include exact -warmup/iteration counts in `measurement`; include the three declared timing -fields both at top level and in `measurement_policy`. Return one actual finite, -positive `samples_ms` value per measured invocation, its median in -`metrics.latency_ms.p50`, and the complete `output_summary` required by the output -contract. Loading happens once before warmup; input preparation and output -materialization follow the declared clock. Metadata validation does not prove -model accuracy or make an unexecuted reference qualified. - -## DataFrame forecast formatting - -Install `pandas` separately (`pip install pandas`) when using the optional table -helpers. They do not load a model or run Python inference: - -```python -import pandas as pd -from trtmc_benchmark.dataframe import prepare_forecast_frame, format_forecast_frame - -frame = pd.read_csv("history.csv", parse_dates=["ds"]) -prepared = prepare_forecast_frame(frame, freq="D") -# prepared.request is the existing native worker's batch solve payload. -# Pass it as request= to an existing resolved batch forecast case. -``` - -Input columns default to `unique_id`, `ds`, and `values`. Series keep first-seen -order; time is sorted within each series. Timestamps must match the explicit -calendar frequency: gaps/duplicates are errors, not silently resampled data. -Use `value_columns=("x", "y")` for one two-channel series, not two batch items. -Missing values retain their masks; no shared imputation or normalization occurs. -`config_by_series={"sensor-a": {"frequency": 0}}` passes family Config unchanged; -calendar `freq` never selects a model's frequency category. - -For an existing resolved `case` whose bundle declares one of -`batch_series_to_point_forecast`, `batch_series_to_quantile_forecast`, or -`batch_series_to_point_and_quantile_forecast`, reuse the normal worker: - -```python -from pathlib import Path -from trtmc_benchmark.types import MeasurementSpec -from trtmc_benchmark.worker import find_worker, run_worker - -case = case.with_values(request=prepared.request, - measurement=MeasurementSpec(warmup=0, iterations=1)) -output_dir = Path("forecast-results") -output_dir.mkdir(parents=True, exist_ok=True) -receipt = run_worker(case, output_dir, find_worker()) -forecast = format_forecast_frame(prepared, receipt["output_summary"]) -``` - -The case retains its explicit runtime root. This example executes one complete -native batch; normal benchmark warmup/iteration settings repeat that batch. -Scalar-only routes reject the batch payload rather than retrying each series. -Real family native-batch support must be qualified separately; this helper does -not imply that the existing TimesFM batch-one bundle supports native batching. - -The result keeps series/horizon/channel order, actual horizon offsets, output -channel labels/units, and all actual quantile levels. The `forecast` column is -the real point output, never a substituted median; quantile-only results have -no invented point column. Unknown output labels remain missing. Heterogeneous -quantile columns have missing cells where a series did not predict that level, -with actual per-series levels recorded in `forecast.attrs`. - -## Timing contract - -Candidate measurements use one scope: public_task_call_wall. - -The candidate runs each family's production Task implementation as loaded. The -benchmark does not override family runtime policy with tuning switches such as -CUDA Graph enablement. - -The new architecture transfers control directly from the public Task interface -to the family implementation, so the old pipeline-call and model-call scopes are -the same boundary. Bundle loading, worker startup, warmup, telemetry, report -generation, and bundle building are excluded. Asset loading is excluded unless a -case explicitly sets asset_loading_included to true. - -The release suite defaults to three warmups and ten measured iterations. A -reference within 5% of candidate p50 is equivalent. Candidate and reference -failures are operational failures, not red performance results. - -## Commands - -Point the environment at one installed runtime directory containing the runtime, -TensorRT backend, and selected family DSOs: - -~~~bash -export TRTMC_PERF_WORKER=/opt/trtmc/bin/trtmc_benchmark_worker -export TRTMC_PERF_RUNTIME_ROOT=/opt/trtmc/lib -export TRTMC_PERF_BUNDLE_CACHE=/data/trtmc-bundles -export TRTMC_PERF_BUNDLE_ROOTS=/data/prebuilt-bundles -export TRTMC_ELF_REFERENCE_REPO=/opt/references/ELF -export TRTMC_LANCE_REFERENCE_REPO=/opt/references/Lance -export TRTMC_SANA_WM_REFERENCE_REPO=/opt/references/Sana -export TRTMC_SANA_WM_MODEL_DIR=/models/sana-wm -export PERSONAPLEX_OFFICIAL_REPO=/opt/references/personaplex -export TRTMC_FAST_FOUNDATION_STEREO_MODEL_DIR=/models/fast-foundation-stereo - -python3 tools/perf_matrix.py check \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml - -python3 tools/perf_matrix.py run \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ - --entry gpt2.generate -~~~ - -Bundle preparation is a separate, untimed step: - -~~~bash -python3 tools/perf_matrix.py prepare \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ - --entry gpt2.generate \ - --output artifacts/perf/bundle-preparation.json -~~~ - -Continue an interrupted run and regenerate its report: - -~~~bash -python3 tools/perf_matrix.py resume artifacts/perf/ -python3 tools/perf_matrix.py report artifacts/perf/ -~~~ - -## Configuration - -The release suite and optional family-owned suites declare model, testcase, -operation, measurement, reference and comparison semantics. Machine paths live -in the existing environment YAML. - -A release entry has one explicit testcase: - -~~~yaml -- id: gpt2.generate - family: gpt2 - operation: generate - model: distilgpt2 - workload: - testcase: distilgpt2 - baseline: - runner: hf-transformers - mode: torch-compile -~~~ - -The catalog reads families/*/tests/manifests/*.json directly. It has no second -model registry; optional performance declarations reuse those exact manifests. -A task that the worker does not implement is reported as unsupported. - -The entry, model, and model-selection options select work. resume continues -incomplete rows, and report regenerates JSON and HTML from stored measurements. - -Reference runners are separate processes so their dependencies never enter the -candidate worker or core runtime. The benchmark-owned cache receipt protects -managed build reuse without changing the public bundle format. It does not -certify externally supplied bundle bytes. - -HTML reports include p50, p95, task rates, measurement boundaries, reproduction -commands, and recorded evidence. Historical v1 results can be rendered without -relabelling their original timing scope; they cannot be resumed as v2 runs. -Historical latency deltas appear only when the recorded comparison identities -match. A performance result does not establish model correctness. - -External reference checkouts must provide their official runtime dependencies. -In particular, PersonaPlex requires the real `sphn` package and Lance requires -the dependencies imported by its upstream `inference_lance.py`; the benchmark -does not install substitutes for either reference. Fast Foundation Stereo also -requires the image and OpenCV packages imported by its official source tree. - -Install benchmark-only dependencies without adding them to a model family: - -```bash -python -m pip install -r apps/benchmark/performance/requirements.txt -``` - -## Internal Accuracy and Performance qualification - -The installed `trtmc-bench` remains a user application. Repository CI and QA -use the separate, non-packaged `tools/model_benchmark.py` driver. A family opts -in by adding one YAML file under `families//tests/benchmark/`; the file -may contain multiple `accuracy` and `performance` cases. Families without that -file, including L0-only models, are not discovered. - -Run every discovered case or select an exact model: - -```bash -python3 tools/model_benchmark.py list -python3 tools/model_benchmark.py run --all --kind performance \ - --runtime-root /opt/trtmc/lib --worker /opt/trtmc/bin/trtmc_benchmark_worker -python3 tools/model_benchmark.py run --model gpt2-125m --kind accuracy \ - --dataset mmlu-five-shot=/data/mmlu_dataset.json \ - --runtime-root /opt/trtmc/lib --worker /opt/trtmc/bin/trtmc_benchmark_worker -``` - -Manual or restricted datasets are supplied as `--dataset ID=PATH`; public -download definitions may instead materialize into `--data-root`. Model files do -not select a GPU. Both Accuracy and Performance invoke the installed -`trtmc-bench` through its public bundle and Task path. Performance tries the -declared compiled reference first and uses eager only when that reference fails -to execute. A completed reference whose output disagrees with the candidate is -a failed contract and never triggers fallback. diff --git a/apps/benchmark/performance/baselines/elf_reference.py b/apps/benchmark/performance/baselines/elf_reference.py deleted file mode 100644 index 11790c49c1..0000000000 --- a/apps/benchmark/performance/baselines/elf_reference.py +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Run the official PyTorch ELF implementation with replayable sampling inputs.""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from pathlib import Path - -import numpy as np -import torch - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--reference-repo", required=True) - parser.add_argument("--config", required=True) - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--dataset", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--shared-inputs-dir", required=True) - parser.add_argument( - "--generation-mode", choices=("conditional", "unconditional"), required=True - ) - parser.add_argument("--sampling-method", choices=("ode", "sde"), required=True) - parser.add_argument("--num-steps", type=int, required=True) - parser.add_argument("--cfg-scale", type=float, required=True) - parser.add_argument("--self-cond-cfg-scale", type=float, required=True) - parser.add_argument("--sde-gamma", type=float, default=0.0) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), default="bf16") - parser.add_argument("--initial-latents", type=Path) - parser.add_argument("--sampling-steps", type=Path) - parser.add_argument("--condition-latents", type=Path) - parser.add_argument("--condition-mask", type=Path) - parser.add_argument("--sde-noises", type=Path) - parser.add_argument("--local-files-only", action="store_true") - return parser.parse_args() - - -def _load_rows(path: Path) -> list[dict]: - return [ - json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip() - ] - - -def _sampling_steps(torch_mod, *, count: int, mean: float, std: float, generator): - inner = torch_mod.randn((count - 1,), generator=generator, dtype=torch_mod.float32) - inner = torch_mod.sigmoid(inner * std + mean).sort().values - return torch_mod.cat((torch_mod.zeros(1), inner, torch_mod.ones(1))) - - -def _trim_terminal(token_ids: list[int], terminal_ids: set[int]) -> list[int]: - end = len(token_ids) - while end and token_ids[end - 1] in terminal_ids: - end -= 1 - return token_ids[:end] - - -def _read_float32(path: Path, *, expected: int | None = None) -> torch.Tensor: - values = np.fromfile(path, dtype=np.float32) - if expected is not None and values.size != expected: - raise ValueError(f"{path} contains {values.size} float32 values; expected {expected}") - return torch.from_numpy(values.copy()) - - -def _validate_replay_arguments(args: argparse.Namespace) -> bool: - sampling_paths = (args.initial_latents, args.sampling_steps) - if any(sampling_paths) and not all(sampling_paths): - raise ValueError("--initial-latents and --sampling-steps must be provided together") - condition_paths = (args.condition_latents, args.condition_mask) - if any(condition_paths) and not all(condition_paths): - raise ValueError("--condition-latents and --condition-mask must be provided together") - if args.generation_mode == "conditional" and args.initial_latents and not all(condition_paths): - raise ValueError("conditional replay requires condition latents and condition mask") - return bool(args.initial_latents) - - -class _InferenceOptimizer: - """Accept checkpoint optimizer state without installing training-only packages.""" - - def load_state_dict(self, _state) -> None: - return None - - -def main() -> int: - args = _parse_args() - replay_inputs = _validate_replay_arguments(args) - reference_src = Path(args.reference_repo).resolve() / "src" - sys.path.insert(0, str(reference_src)) - - from configs.config import load_config_from_yaml - from generation import _build_eval_model - from modules.model import ELF_models - from modules.t5_encoder import get_encoder - from utils.checkpoint_utils import load_checkpoint - from utils.data_utils import ( - get_dataloader, - get_pad_token_id, - load_jsonl_dataset, - ) - from utils.encoder_utils import encode_text - from utils.generation_utils import ( - _dlm_decode_batch, - mask_after_eos, - shift_left, - ) - from utils.sampling_utils import _ode_step, _forward_sample, restore_cond - from utils.train_utils import TrainState - - if args.local_files_only: - import os - - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ["TRANSFORMERS_OFFLINE"] = "1" - - config = load_config_from_yaml(args.config) - config.use_bf16 = args.precision == "bf16" - config.use_compile = False - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - if device.type == "cuda": - # cuDNN SDPA can request an impractically large workspace for ELF's - # 1,088-token XSum sequence on constrained-memory CUDA hosts. Prefer - # PyTorch's bounded-memory SDPA implementations for a reference run. - torch.backends.cuda.enable_cudnn_sdp(False) - torch.backends.cuda.enable_flash_sdp(False) - torch.backends.cuda.enable_mem_efficient_sdp(True) - torch.backends.cuda.enable_math_sdp(True) - encoder_model_id = config.encoder_model_name - tokenizer_model_id = config.tokenizer_name or config.encoder_model_name - tokenizer = __import__("transformers").AutoTokenizer.from_pretrained( - tokenizer_model_id, - local_files_only=args.local_files_only, - ) - pad_token_id = get_pad_token_id(tokenizer, config.pad_token) - terminal_ids = {int(pad_token_id), int(tokenizer.eos_token_id)} - - encoder = None - if replay_inputs: - replay_initial = _read_float32(args.initial_latents) - if replay_initial.numel() % int(config.max_length) != 0: - raise ValueError("initial replay latents do not divide evenly by config.max_length") - text_encoder_dim = replay_initial.numel() // int(config.max_length) - elif args.generation_mode == "conditional": - encoder_config, encoder = get_encoder(encoder_model_id, torch.float32) - encoder = encoder.to(device).eval() - for parameter in encoder.parameters(): - parameter.requires_grad_(False) - text_encoder_dim = encoder_config.d_model - else: - text_encoder_dim = 512 - - model = ELF_models[config.model]( - text_encoder_dim=text_encoder_dim, - max_length=config.max_length, - attn_drop=config.attn_dropout, - proj_drop=config.proj_dropout, - num_time_tokens=config.num_time_tokens, - num_self_cond_cfg_tokens=config.num_self_cond_cfg_tokens, - vocab_size=tokenizer.vocab_size, - num_model_mode_tokens=config.num_model_mode_tokens, - bottleneck_dim=config.bottleneck_dim, - ).to(device) - state = TrainState( - model=model, - optimizer=_InferenceOptimizer(), - lr_scheduler=None, - ema_params1=TrainState.init_ema(model), - step=0, - epoch=0, - dropout_generator=torch.Generator(device="cpu").manual_seed(args.seed), - ) - state, _ = load_checkpoint(args.checkpoint, state) - model = _build_eval_model(state, use_compile=False).to(device).eval() - dtype = next(model.parameters()).dtype - autocast_dtype = { - "fp16": torch.float16, - "bf16": torch.bfloat16, - "fp32": torch.float32, - }[args.precision] - - rows = _load_rows(Path(args.dataset)) - conditional_batches = None - if args.generation_mode == "conditional" and not replay_inputs: - dataset = load_jsonl_dataset( - args.dataset, tokenizer, input_key="input", output_key="output" - ) - conditional_batches = iter( - get_dataloader( - dataset, - batch_size=1, - shuffle=False, - num_workers=0, - drop_last=False, - max_seq_length=config.max_length, - pad_token_id=pad_token_id, - max_input_seq_length=config.max_input_length, - distributed=False, - ) - ) - - shared_root = Path(args.shared_inputs_dir) - shared_root.mkdir(parents=True, exist_ok=True) - responses: list[dict] = [] - for index, row in enumerate(rows): - sample_id = str(row.get("id", f"elf_{index:06d}")) - sample_dir = shared_root / sample_id - sample_dir.mkdir(parents=True, exist_ok=True) - if replay_inputs: - latent_count = int(config.max_length) * int(text_encoder_dim) - z = replay_initial.reshape(1, config.max_length, text_encoder_dim).clone() - t_steps = _read_float32(args.sampling_steps, expected=args.num_steps + 1) - sde_noises = [] - if args.sde_noises: - noise = _read_float32(args.sde_noises) - sde_noises = list( - noise.reshape(-1, config.max_length, text_encoder_dim).unbind(0) - ) - if args.generation_mode == "conditional": - cond_seq = _read_float32( - args.condition_latents, expected=latent_count - ).reshape(1, config.max_length, text_encoder_dim) - cond_mask = _read_float32( - args.condition_mask, expected=int(config.max_length) - ).reshape(1, config.max_length) - else: - cond_seq = torch.zeros_like(z) - cond_mask = torch.zeros((1, config.max_length), dtype=torch.float32) - else: - generator = torch.Generator(device="cpu").manual_seed(args.seed + index) - t_steps = _sampling_steps( - torch, - count=args.num_steps, - mean=float(config.denoiser_p_mean), - std=float(config.denoiser_p_std), - generator=generator, - ) - z = torch.randn( - (1, config.max_length, text_encoder_dim), - generator=generator, - dtype=torch.float32, - ) * float(config.denoiser_noise_scale) - sde_noises = [] - if args.sampling_method == "sde": - for _ in range(args.num_steps - 1): - sde_noises.append( - torch.randn(z.shape, generator=generator, dtype=torch.float32) - * float(config.denoiser_noise_scale) - ) - z.numpy().tofile(sample_dir / "initial_latents.f32") - t_steps.numpy().tofile(sample_dir / "sampling_steps.f32") - if sde_noises: - torch.stack(sde_noises).numpy().tofile(sample_dir / "sde_noises.f32") - - if args.generation_mode == "conditional" and not replay_inputs: - batch = next(conditional_batches) - input_ids = torch.from_numpy(np.asarray(batch["input_ids"])).to(device).long() - encoder_mask = ( - torch.from_numpy(np.asarray(batch["encoder_attention_mask"])).to(device).float() - ) - cond_mask = torch.from_numpy(np.asarray(batch["cond_seq_mask"])).to(device).float() - cond_seq = encode_text( - input_ids=input_ids, - attention_mask=encoder_mask, - encoder=encoder, - latent_mean=config.latent_mean, - latent_std=config.latent_std, - ).to(dtype) - elif not replay_inputs: - cond_seq = torch.zeros_like(z, dtype=dtype, device=device) - cond_mask = torch.zeros((1, config.max_length), dtype=dtype, device=device) - - z = z.to(device=device, dtype=dtype) - t_steps = t_steps.to(device=device, dtype=dtype) - cond_seq = cond_seq.to(device=device, dtype=dtype) - cond_mask = cond_mask.to(device=device, dtype=dtype) - z = restore_cond(z, cond_seq, cond_mask) - x_pred = restore_cond(torch.zeros_like(z), cond_seq, cond_mask) - step_kwargs = dict( - model=model, - config=config, - cfg_scale=args.cfg_scale, - self_cond_cfg_scale=args.self_cond_cfg_scale, - cond_seq=cond_seq, - cond_seq_mask=cond_mask, - ) - started = time.perf_counter() - with ( - torch.no_grad(), - torch.amp.autocast( - "cuda", - dtype=autocast_dtype, - enabled=device.type == "cuda" and args.precision != "fp32", - ), - ): - for step_index in range(len(t_steps) - 2): - t = float(t_steps[step_index].item()) - t_next = float(t_steps[step_index + 1].item()) - if args.sampling_method == "ode": - z, x_pred = _ode_step( - z=z, t=t, t_next=t_next, x_pred_prev=x_pred, **step_kwargs - ) - else: - h = t_next - t - alpha = max(0.0, min(1.0, 1.0 - args.sde_gamma * h)) - t_back = alpha * t - eps = sde_noises[step_index].to(device=device, dtype=dtype) - z_back = restore_cond(alpha * z + (1.0 - alpha) * eps, cond_seq, cond_mask) - t_batch = torch.full((1,), t_back, dtype=dtype, device=device) - v_pred, x_pred = _forward_sample( - z=z_back, t_batch=t_batch, x_pred_prev=x_pred, **step_kwargs - ) - z = z_back + (t_next - t_back) * v_pred - z, x_pred = _ode_step( - z=z, - t=float(t_steps[-2].item()), - t_next=float(t_steps[-1].item()), - x_pred_prev=x_pred, - **step_kwargs, - ) - if args.precision == "fp16" and device.type == "cuda": - batch_size = z.shape[0] - t_final = torch.full( - (batch_size,), float(t_steps[-1].item()), dtype=z.dtype, device=device - ) - scale = ( - torch.full( - (batch_size,), args.self_cond_cfg_scale, dtype=z.dtype, device=device - ) - if config.num_self_cond_cfg_tokens > 0 - else None - ) - z_input = torch.cat([z, torch.zeros_like(z)], dim=-1) - with torch.amp.autocast("cuda", dtype=torch.float16): - _, decoder_logits = model( - z_input, - t_final, - deterministic=True, - self_cond_cfg_scale=scale, - decoder_step_active=True, - ) - predicted = decoder_logits.argmax(dim=-1) - else: - predicted = _dlm_decode_batch( - z, model, float(t_steps[-1].item()), config, args.self_cond_cfg_scale - ) - if args.generation_mode == "conditional": - cond_len = cond_mask.to(torch.int32).sum(dim=1) - predicted = shift_left(predicted, cond_len, 0)[ - :, : config.max_length - config.max_input_length - ] - predicted = mask_after_eos( - predicted, eos_token_id=tokenizer.eos_token_id, pad_token_id=pad_token_id - ) - token_ids = _trim_terminal( - [int(token_id) for token_id in predicted[0].detach().cpu().tolist()], - terminal_ids, - ) - responses.append( - { - "sample_id": sample_id, - "output_text": tokenizer.decode(token_ids, skip_special_tokens=True).strip(), - "generated_token_ids": token_ids, - "wall_ms": (time.perf_counter() - started) * 1000.0, - "source": "hf_elf_torch", - "shared_inputs_dir": str(sample_dir), - "shared_sampling_inputs": { - "initial_latents": str( - args.initial_latents or sample_dir / "initial_latents.f32" - ), - "sampling_steps": str( - args.sampling_steps or sample_dir / "sampling_steps.f32" - ), - **({"sde_noises": str(sample_dir / "sde_noises.f32")} if sde_noises else {}), - }, - } - ) - - Path(args.output).write_text( - json.dumps({"responses": responses}, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/benchmark/performance/baselines/reference_support.py b/apps/benchmark/performance/baselines/reference_support.py deleted file mode 100644 index 3bebc0710a..0000000000 --- a/apps/benchmark/performance/baselines/reference_support.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Reference-only helpers kept outside Model Connect core and families.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import numpy as np - - -def preprocess_locateanything(image_path: str) -> dict[str, np.ndarray]: - from PIL import Image - - size = 448 - patch = 14 - image = Image.open(image_path).convert("RGB").resize((size, size), Image.Resampling.BICUBIC) - pixels = np.asarray(image, dtype=np.float32) / 255.0 - pixels = (pixels - np.asarray((0.5, 0.5, 0.5), dtype=np.float32)) / np.asarray( - (0.5, 0.5, 0.5), dtype=np.float32 - ) - chw = pixels.transpose(2, 0, 1) - grid = size // patch - patches = ( - chw.reshape(3, grid, patch, grid, patch) - .transpose(1, 3, 0, 2, 4) - .reshape(grid * grid, 3, patch, patch) - ) - return { - "pixel_values": patches.astype(np.float32), - "image_grid_hws": np.asarray([[grid, grid]], dtype=np.int32), - } - - -def configure_official_model_args( - model: Any, *, max_disparity: int, valid_iters: int -) -> None: - model.args.max_disp = max_disparity - model.args.valid_iters = valid_iters - model.args.normalize = True - - -def fast_foundation_stereo_model_dir(configured: str) -> Path: - root = Path(configured).expanduser().resolve() - required = ( - root / "core/foundation_stereo.py", - root / "core/submodule.py", - root / "weights/23-36-37/model_best_bp2_serialize.pth", - ) - if not all(path.is_file() for path in required): - raise RuntimeError("Fast Foundation Stereo reference model directory is incomplete") - return root diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py deleted file mode 100644 index a94c51d70d..0000000000 --- a/apps/benchmark/performance/baselines/task_reference.py +++ /dev/null @@ -1,3240 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Benchmark task-aligned non-text model reference operations.""" - -from __future__ import annotations - -import argparse -import atexit -from dataclasses import dataclass -from datetime import datetime, timezone -import json -import importlib -import math -import os -from pathlib import Path -import platform -import shutil -import statistics -import subprocess -import sys -import tempfile -import time -from typing import Any, Callable, Mapping, Sequence - - -REPOSITORY = Path(__file__).resolve().parents[4] -for source_root in ( - REPOSITORY, - REPOSITORY / "core/builder", - REPOSITORY / "apps/benchmark", -): - if str(source_root) not in sys.path: - sys.path.insert(0, str(source_root)) - -from apps.benchmark.performance.baselines.timing_contracts import timing_contract # noqa: E402 -from apps.benchmark.performance.baselines.hf_transformers import _batch_prompt, flatten_config # noqa: E402 - -SYSTEM_PROMPT_QWEN3_OMNI = ( - "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " - "capable of perceiving auditory and visual inputs, as well as generating " - "text and speech." -) -QWEN3_OMNI_TEXT_CHAT_TEMPLATE = """{%- for message in messages %} -{{- '<|im_start|>' + message.role + '\n' }} -{%- if message.content is string %} -{{- message.content }} -{%- else %} -{%- for item in message.content %} -{%- if item.type == 'text' %}{{- item.text }}{%- endif %} -{%- endfor %} -{%- endif %} -{{- '<|im_end|>\n' }} -{%- endfor %} -{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}""" -MAGPIE_SPEAKER_ENCODER_REPO = "Edresson/Speaker_Encoder_H_ASP" -MAGPIE_SPEAKER_ENCODER_FILENAME = "pytorch_model.bin" -MAGPIE_SPEAKER_ENCODER_URL = ( - "https://huggingface.co/Edresson/Speaker_Encoder_H_ASP/resolve/main/pytorch_model.bin" -) -# The official offline example default, matching the current native profile. -VOICECHAT_SYSTEM_PROMPT = ( - "You are an AI voice assistant developed by NVIDIA. Your name is NVIDIA Voice Chat. " - "Answer in a spoken, conversational style rather than a written one. Do not repeat " - "the same sentence over and over again. Start the conversation by greeting the user." -) -ADAPTERS = ( - "hf-diffusers", - "hf-diffusers-modular", - "hf-chat-asr", - "hf-qwen3-omni", - "hf-transformers-asr", - "hf-transformers-embedding", - "hf-transformers-reranking", - "hf-transformers-tts", - "hf-transformers-vision", - "hf-transformers-vlm", - "nemo-asr", - "nemo-tts", - "nemo-voicechat", - "pytorch-lerobot-act", - "pytorch-personaplex", - "pytorch-timeseries", - "timm-classification", - "upstream-elf", - "upstream-fast-foundation-stereo", - "upstream-lance", - "upstream-moge", - "upstream-sana-wm", -) -PYTORCH_ADAPTERS = { - "nemo-asr", - "nemo-tts", - "nemo-voicechat", - "pytorch-lerobot-act", - "pytorch-personaplex", - "pytorch-timeseries", - "upstream-elf", - "upstream-fast-foundation-stereo", - "upstream-lance", - "upstream-moge", - "upstream-sana-wm", -} - - -@dataclass(frozen=True) -class Session: - """One loaded reference model and its repeatable timed operation.""" - - invoke: Callable[[], Any] - framework: str - timing_scope: str = "task-model-call-wall" - input_preparation_included: bool = False - asset_loading_included: bool = False - compile_evidence: dict[str, Any] | None = None - summarize: Callable[[Any], Mapping[str, Any]] | None = None - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--adapter", required=True, choices=ADAPTERS) - parser.add_argument("--family", required=True) - parser.add_argument("--operation", required=True) - parser.add_argument("--model", required=True) - parser.add_argument("--revision") - parser.add_argument("--manifest", required=True, type=Path) - parser.add_argument( - "--selected-task", help="Semantic Task selected independently of bundle identity" - ) - parser.add_argument("--request-json", required=True) - parser.add_argument("--adapter-options-json", default="{}") - parser.add_argument("--timing-contract-json", default="{}") - parser.add_argument("--precision", required=True, choices=("fp16", "fp32", "bf16")) - parser.add_argument( - "--mode", required=True, choices=("hf-eager", "pytorch-eager", "torch-compile") - ) - parser.add_argument("--padding", default="longest") - parser.add_argument("--trust-remote-code", action="store_true") - parser.add_argument("--local-files-only", action="store_true") - parser.add_argument("--warmup", required=True, type=int) - parser.add_argument("--iterations", required=True, type=int) - parser.add_argument("--case-name", required=True) - parser.add_argument( - "--testcase-name", help="Selected manifest testcase, not the performance entry ID" - ) - parser.add_argument("--output", required=True, type=Path) - return parser - - -def _json_object(raw: str, label: str) -> dict[str, Any]: - try: - value = json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError(f"{label} is not valid JSON: {exc}") from exc - if not isinstance(value, dict): - raise ValueError(f"{label} must contain an object") - return value - - -def _selected_task(arguments: argparse.Namespace) -> str: - selected = getattr(arguments, "selected_task", None) - if selected is None: - return json.loads(arguments.manifest.read_text(encoding="utf-8"))["task"] - if not isinstance(selected, str) or not selected or selected != selected.strip(): - raise ValueError("selected_task must be a nonempty Task ID without surrounding whitespace") - return selected - - -def _reference_checkout(checkout: str, *, repository: str) -> str: - path = Path(checkout).resolve() - if not path.is_dir(): - raise ValueError(f"{repository} checkout does not exist: {path}") - return str(path) - - -def _torch_dtype(torch_module: Any, precision: str) -> Any: - return { - "fp16": torch_module.float16, - "fp32": torch_module.float32, - "bf16": torch_module.bfloat16, - }[precision] - - -def _seed_all(torch_module: Any, seed: int) -> None: - import random - - import numpy as np - - random.seed(seed) - np.random.seed(seed) - torch_module.manual_seed(seed) - if torch_module.cuda.is_available(): - torch_module.cuda.manual_seed_all(seed) - - -def _request_seed(request: Mapping[str, Any], default: Any = 42) -> int: - value = request.get("seed", default) - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError("reference request seed must be an integer") - return value - - -def _bark_generation_options(request: Mapping[str, Any]) -> dict[str, int]: - max_new_tokens = request.get("max_new_tokens", 0) - if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): - raise ValueError("Bark request max_new_tokens must be an integer") - if max_new_tokens <= 0: - return {} - return {"semantic_max_new_tokens": max_new_tokens} - - -def _apply_magpie_generation_options(model: Any, request: Mapping[str, Any]) -> None: - max_new_tokens = request.get("max_new_tokens", 0) - if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): - raise ValueError("Magpie request max_new_tokens must be an integer") - if max_new_tokens > 0: - model.inference_parameters.max_decoder_steps = max_new_tokens - - -def _load_kwargs(arguments: argparse.Namespace, torch_module: Any) -> dict[str, Any]: - values: dict[str, Any] = { - "trust_remote_code": arguments.trust_remote_code, - "local_files_only": arguments.local_files_only, - "torch_dtype": _torch_dtype(torch_module, arguments.precision), - } - if arguments.revision: - values["revision"] = arguments.revision - return values - - -def _processor_kwargs(arguments: argparse.Namespace) -> dict[str, Any]: - values: dict[str, Any] = { - "trust_remote_code": arguments.trust_remote_code, - "local_files_only": arguments.local_files_only, - } - if arguments.revision: - values["revision"] = arguments.revision - return values - - -def _load_sam3_processor( - transformers: Any, - model: str, - processor_kwargs: Mapping[str, Any], -) -> Any: - return transformers.Sam3Processor.from_pretrained(model, **processor_kwargs) - - -def _cached_snapshot_path(repo_id: str, requested: str | None, marker_file: str) -> Path | None: - if Path(repo_id).exists(): - return Path(repo_id).resolve() - try: - from huggingface_hub import try_to_load_from_cache - - cached = try_to_load_from_cache( - repo_id=repo_id, - filename=marker_file, - revision=requested or "main", - ) - except (ImportError, OSError, ValueError): - return None - if isinstance(cached, str) and Path(cached).is_file(): - return Path(cached).parent.resolve() - return None - - -def _to_device(value: Any, device: Any, dtype: Any = None) -> Any: - if isinstance(value, Mapping): - return {name: _to_device(item, device, dtype) for name, item in value.items()} - if hasattr(value, "to"): - if dtype is not None and getattr(value, "is_floating_point", lambda: False)(): - return value.to(device=device, dtype=dtype) - return value.to(device) - return value - - -def _asset_path(arguments: argparse.Namespace, request: Mapping[str, Any], key: str) -> Path: - raw = str(request.get(key, "") or "") - if not raw: - raise ValueError(f"{arguments.family} reference requires request.{key}") - path = Path(raw) - if not path.is_absolute(): - path = arguments.manifest.resolve().parent.parent / path - if not path.is_file(): - raise FileNotFoundError(f"reference input does not exist: {path}") - return path - - -def _tensor_summary(value: Any) -> dict[str, Any]: - shape = [int(dim) for dim in value.shape] - return { - "shape": shape, - "element_count": int(value.numel()), - "finite": bool(value.isfinite().all().item()), - } - - -def _flatten_tensor_values(value: Any) -> list[float]: - flattened: list[float] = [] - - def visit(item: Any) -> None: - if isinstance(item, (list, tuple)): - for child in item: - visit(child) - else: - flattened.append(float(item)) - - visit(value.detach().float().cpu().tolist()) - return flattened - - -def _forecast_summary( - value: Any, task: str, quantile_levels: Sequence[float] = () -) -> dict[str, Any]: - summary = _tensor_summary(value) - summary["values"] = _flatten_tensor_values(value) - if task not in {"series_to_point_forecast", "series_to_quantile_forecast"}: - return summary - shape = summary["shape"] - if len(shape) not in {2, 3} or shape[0] != 1: - raise ValueError("forecast reference requires an explicit one-series batch axis") - if task == "series_to_point_forecast": - horizon, channels = shape[1], shape[2] if len(shape) == 3 else 1 - summary.update(shape=[horizon, channels], axes=["horizon", "channel"]) - else: - if len(shape) != 3 or len(quantile_levels) != shape[1]: - raise ValueError( - "quantile forecast reference requires exact checkpoint quantile levels" - ) - horizon = shape[2] - summary.update( - shape=[shape[1], horizon, 1], - axes=["quantile", "horizon", "channel"], - quantile_levels=list(quantile_levels), - ) - summary.update( - forecast_elements=summary["element_count"], horizon_steps=list(range(1, horizon + 1)) - ) - return summary - - -def _regression_values_summary(value: Any) -> dict[str, Any]: - summary = _tensor_summary(value) - shape = summary["shape"] - if len(shape) != 2 or shape[0] != 1 or shape[1] <= 0: - raise ValueError("deterministic regression requires one batch with a nonempty target axis") - if not summary["finite"]: - raise ValueError("deterministic regression target values must be finite") - return { - "kind": "regression_values", - "target_count": shape[1], - "regression_targets": shape[1], - "parameter_elements": 0, - "values": value[0].detach().float().cpu().tolist(), - "axes": ["target"], - "target_names": [], - "target_units": [], - } - - -def _regression_summary( - output: Any, distribution: str, parameter_names: Sequence[str] -) -> dict[str, Any]: - if distribution not in {"normal", "student_t", "negative_binomial"}: - raise ValueError("regression reference requires a declared probability distribution") - if not isinstance(output, (tuple, list)) or len(output) != len(parameter_names) or not output: - raise ValueError("regression reference parameters do not match the declared distribution") - # Transformers names distribution parameters loc/df. The public Task - # contract names them location/degrees_of_freedom; never infer tuple order. - aliases = {"loc": "location", "df": "degrees_of_freedom"} - names = [aliases.get(name, name) for name in parameter_names] - required = { - "normal": {"location", "scale"}, - "student_t": {"degrees_of_freedom", "location", "scale"}, - "negative_binomial": {"total_count", "logits"}, - }[distribution] - if len(names) != len(required) or set(names) != required: - raise ValueError( - "regression reference parameter names do not match the declared distribution" - ) - parameters = [] - targets = None - for name, value in zip(names, output): - shape = tuple(int(size) for size in value.shape) - if ( - len(shape) != 2 - or shape[0] != 1 - or shape[1] <= 0 - or (targets is not None and shape[1] != targets) - ): - raise ValueError( - "regression parameters must each have one batch and the same target axis" - ) - targets = shape[1] - if not _tensor_summary(value)["finite"]: - raise ValueError("regression reference returned nonfinite parameters") - parameters.append({"name": name, "values": value[0].detach().float().cpu().tolist()}) - return { - "distribution": distribution, - "target_count": targets, - "regression_targets": targets, - "parameter_elements": targets * len(parameters), - "axes": ["target"], - "parameters": parameters, - "target_names": [], - "target_units": [], - } - - -def _task_value( - arguments: argparse.Namespace, - request: Mapping[str, Any], - name: str, - default: Any = None, -) -> Any: - if name in request: - return request[name] - return default - - -def _load_tts( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") - import torch - - torch.use_deterministic_algorithms(True) - device = torch.device("cuda") - prompt = str(request.get("prompt", "")) - if arguments.adapter == "nemo-tts": - import fsspec - from huggingface_hub import hf_hub_download - from nemo.collections.tts.models import MagpieTTSModel - - speaker_revision = str(options.get("speaker_encoder_revision", "") or "") - if not speaker_revision: - raise ValueError("Magpie TTS requires adapter_options.speaker_encoder_revision") - speaker_checkpoint = hf_hub_download( - repo_id=MAGPIE_SPEAKER_ENCODER_REPO, - filename=MAGPIE_SPEAKER_ENCODER_FILENAME, - revision=speaker_revision, - local_files_only=arguments.local_files_only, - ) - original_fsspec_open = fsspec.open - - def offline_fsspec_open(path: Any, *args: Any, **kwargs: Any) -> Any: - normalized = str(path).split("?", 1)[0] - if normalized == MAGPIE_SPEAKER_ENCODER_URL: - path = speaker_checkpoint - return original_fsspec_open(path, *args, **kwargs) - - fsspec.open = offline_fsspec_open - model_archive = hf_hub_download( - repo_id=arguments.model, - filename="magpie_tts_multilingual_357m.nemo", - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - model = MagpieTTSModel.restore_from(restore_path=model_archive).eval().to(device) - _apply_magpie_generation_options(model, request) - seed = _request_seed( - request, - 42, - ) - - def invoke() -> Mapping[str, Any]: - _seed_all(torch, seed) - with torch.inference_mode(): - audio, length = model.do_tts(transcript=prompt, language="en", use_cfg=True) - sample_count = int(length.item()) if length.numel() else int(audio.numel()) - return { - "audio_samples": sample_count, - "sample_rate": 22_050, - "_audio_f32": audio.detach().float().cpu().reshape(-1)[:sample_count].numpy(), - } - - else: - from transformers import AutoProcessor, BarkModel - - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - BarkModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - inputs = _to_device(processor(prompt, return_tensors="pt"), device) - seed = _request_seed( - request, - 42, - ) - generation_options = _bark_generation_options(request) - - def invoke() -> Mapping[str, Any]: - _seed_all(torch, seed) - with torch.inference_mode(): - audio = model.generate(**inputs, **generation_options) - return { - "audio_samples": int(audio.numel()), - "sample_rate": int(model.generation_config.sample_rate), - "_audio_f32": audio.detach().float().cpu().reshape(-1).numpy(), - } - - if arguments.adapter == "nemo-tts": - return Session( - invoke, - "nemo", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - return Session(invoke, "transformers") - - -def _load_nemo_asr_reference_model( - arguments: argparse.Namespace, - options: Mapping[str, Any], - *, - device: Any, -) -> Any: - loader = str(options.get("model_loader", "nemo-pretrained")) - if loader == "prompted-nemo-archive": - from apps.benchmark.performance.baselines.audio_reference import ( - load_nemotron35_asr_model, - ) - - return load_nemotron35_asr_model( - model=arguments.model, - revision=arguments.revision or "", - local_files_only=arguments.local_files_only, - device=str(device), - ) - if loader != "nemo-pretrained": - raise ValueError(f"unsupported NeMo ASR model_loader: {loader}") - - import nemo.collections.asr as nemo_asr - - return nemo_asr.models.ASRModel.from_pretrained( - arguments.model, - map_location="cpu", - ) - - -def _disable_nemo_asr_cuda_graphs(model: Any) -> bool: - """Disable NeMo's optional RNNT CUDA Graph decoder when available.""" - decoding = getattr(getattr(model, "decoding", None), "decoding", None) - disable_cuda_graphs = getattr(decoding, "disable_cuda_graphs", None) - if not callable(disable_cuda_graphs): - return False - return bool(disable_cuda_graphs()) - - -def _load_asr( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import torch - from apps.benchmark.performance.baselines.audio_reference import ( - read_wav_float32, - resample_audio, - transcription_text, - write_wav_pcm16, - ) - - audio, sample_rate = read_wav_float32(str(_asset_path(arguments, request, "audio_path"))) - target_rate = 16_000 - audio = resample_audio(audio, sample_rate, target_rate) - max_new_tokens = int(request.get("max_new_tokens", 100)) - device = torch.device("cuda") - - if arguments.adapter == "hf-chat-asr": - from transformers import AutoModel, AutoProcessor - - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - AutoModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - conversation = [ - { - "role": "user", - "content": [ - {"type": "audio", "audio": audio}, - {"type": "text", "text": "Please transcribe this audio into text"}, - ], - } - ] - inputs = processor.apply_chat_template( - conversation, - tokenize=True, - add_generation_prompt=True, - return_tensors="pt", - return_dict=True, - sampling_rate=target_rate, - ).to(model.device) - prompt_tokens = int(inputs["input_ids"].shape[1]) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - generated = model.generate( - **inputs, - max_new_tokens=max_new_tokens, - do_sample=False, - ) - token_ids = [int(token) for token in generated[0, prompt_tokens:].cpu().tolist()] - return { - "text": processor.tokenizer.decode(token_ids, skip_special_tokens=True).strip(), - "token_ids": token_ids, - "output_tokens": len(token_ids), - } - - elif arguments.adapter == "nemo-asr": - model = _load_nemo_asr_reference_model(arguments, options, device=device).eval().to(device) - _disable_nemo_asr_cuda_graphs(model) - reference_dtype = _torch_dtype(torch, arguments.precision) - autocast_dtype = reference_dtype if arguments.precision != "fp32" else torch.float16 - temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) - temporary.close() - atexit.register(Path(temporary.name).unlink, missing_ok=True) - write_wav_pcm16(Path(temporary.name), audio, target_rate) - - transcription_input: str | list[str] - transcription_format = str(options.get("transcription_input", "audio-files")) - if transcription_format == "manifest": - manifest = tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) - manifest.close() - atexit.register(Path(manifest.name).unlink, missing_ok=True) - record: dict[str, Any] = { - "audio_filepath": temporary.name, - "duration": float(len(audio)) / target_rate, - "text": "", - } - language = str(request.get("language", "") or "") - if language and language != "auto": - record["lang"] = language - Path(manifest.name).write_text(json.dumps(record) + "\n", encoding="utf-8") - transcription_input = manifest.name - elif transcription_format == "audio-files": - transcription_input = [temporary.name] - else: - raise ValueError( - f"unsupported NeMo ASR transcription_input: {transcription_format}" - ) - - def invoke() -> Mapping[str, Any]: - with torch.autocast( - device_type="cuda", - dtype=autocast_dtype, - enabled=arguments.precision != "fp32", - ): - result = model.transcribe(transcription_input, batch_size=1) - return {"text": transcription_text(result), "output_tokens": None} - - else: - from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor - - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - AutoModelForSpeechSeq2Seq.from_pretrained( - arguments.model, **_load_kwargs(arguments, torch) - ) - .eval() - .to(device) - ) - processor_options: dict[str, Any] = { - "sampling_rate": target_rate, - "return_tensors": "pt", - } - language = str(request.get("language", "") or "") - if language: - processor_options["language"] = language - inputs = _to_device( - processor(audio, **processor_options), device, next(model.parameters()).dtype - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - generated = model.generate(**inputs, max_new_tokens=max_new_tokens) - sequences = generated.sequences if hasattr(generated, "sequences") else generated - token_ids = [int(token) for token in sequences[0].detach().cpu().tolist()] - return { - "text": processor.batch_decode(sequences, skip_special_tokens=True)[0], - "token_ids": token_ids, - "output_tokens": len(token_ids), - } - - framework = "nemo" if arguments.adapter == "nemo-asr" else "transformers" - if arguments.adapter == "nemo-asr": - return Session( - invoke, - framework, - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=True, - ) - return Session(invoke, framework) - - -def _load_vlm_model( - transformers_module: Any, class_name: str, model_id: str, kwargs: dict[str, Any] -) -> Any: - if class_name not in { - "AutoModel", - "AutoModelForCausalLM", - "AutoModelForImageTextToText", - }: - raise ValueError(f"unsupported vision-language model_class: {class_name}") - model_class = getattr(transformers_module, class_name) - return model_class.from_pretrained(model_id, **kwargs) - - -def _load_deepseek_ocr( - arguments: argparse.Namespace, - request: Mapping[str, Any], -) -> Session: - import torch - from transformers import AutoModel, AutoTokenizer - - if arguments.precision != "bf16": - raise ValueError("DeepSeek-OCR-2 official reference requires bf16 precision") - tokenizer = AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - load_options = _load_kwargs(arguments, torch) - load_options.update( - { - "attn_implementation": "eager", - "use_safetensors": True, - } - ) - model = AutoModel.from_pretrained(arguments.model, **load_options).eval().cuda() - image_path = str(_asset_path(arguments, request, "image_path")) - prompt = str(request.get("prompt", "")) - if "" not in prompt: - prompt = f"\n{prompt}" - output_directory = Path(tempfile.mkdtemp(prefix="trtmc-perf-deepseek-ocr-")) - atexit.register(shutil.rmtree, output_directory, ignore_errors=True) - - def invoke() -> Mapping[str, Any]: - text = model.infer( - tokenizer, - prompt=prompt, - image_file=image_path, - output_path=str(output_directory), - base_size=1024, - image_size=768, - crop_mode=True, - save_results=False, - eval_mode=True, - ) - decoded = str(text or "") - token_ids = tokenizer(decoded, add_special_tokens=False).input_ids - return {"text": decoded, "token_ids": token_ids, "output_tokens": len(token_ids)} - - return Session( - invoke, - "transformers", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=True, - ) - - -def _locateanything_config(arguments: argparse.Namespace, transformers_module: Any) -> Any: - options = _processor_kwargs(arguments) - return transformers_module.AutoConfig.from_pretrained(arguments.model, **options) - - -def _locateanything_tokenizer(arguments: argparse.Namespace, _torch_module: Any) -> Any: - from transformers import AutoTokenizer - - return AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - - -def _load_locateanything( - arguments: argparse.Namespace, - request: Mapping[str, Any], -) -> Session: - import torch - import transformers - from apps.benchmark.performance.baselines.reference_support import ( - preprocess_locateanything, - ) - - if torch.cuda.is_available(): - torch.backends.cudnn.enabled = False - device = torch.device("cuda") - config = _locateanything_config(arguments, transformers) - tokenizer = _locateanything_tokenizer(arguments, torch) - load_options = _load_kwargs(arguments, torch) - load_options["config"] = config - model = ( - transformers.AutoModel.from_pretrained(arguments.model, **load_options).eval().to(device) - ) - image_inputs = preprocess_locateanything(str(_asset_path(arguments, request, "image_path"))) - pixel_values = torch.from_numpy(image_inputs["pixel_values"]).to(device) - image_grid_hws = torch.from_numpy(image_inputs["image_grid_hws"]).to( - device=device, dtype=torch.int32 - ) - prompt = str(request.get("prompt", "")) - prompt_text = ( - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - "<|im_start|>user\n" - + "" * 256 - + f"{prompt}<|im_end|>\n<|im_start|>assistant\n" - ) - inputs = tokenizer(prompt_text, return_tensors="pt") - input_ids = inputs["input_ids"].to(device) - generate_options: dict[str, Any] = { - "pixel_values": pixel_values, - "image_grid_hws": image_grid_hws, - "input_ids": input_ids, - "tokenizer": tokenizer, - "max_new_tokens": int(request.get("max_new_tokens", 32)), - "use_cache": True, - "generation_mode": "slow", - "do_sample": False, - } - if inputs.get("attention_mask") is not None: - generate_options["attention_mask"] = inputs["attention_mask"].to(device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - output = model.generate(**generate_options) - if isinstance(output, str): - text = output - token_ids = tokenizer.encode(text, add_special_tokens=False) - elif isinstance(output, (list, tuple)) and output and isinstance(output[0], str): - text = output[0] - token_ids = tokenizer.encode(text, add_special_tokens=False) - else: - sequence = output[0] if output.ndim > 1 else output - generated = sequence[input_ids.shape[-1] :] - token_ids = [int(token) for token in generated.detach().cpu().tolist()] - text = tokenizer.decode(token_ids, skip_special_tokens=True) - if not text.strip(): - raise RuntimeError("LocateAnything reference produced empty text") - return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} - - return Session(invoke, "transformers") - - -def _vl_prompt_has_image_placeholder(text: str) -> bool: - return any( - marker in text - for marker in ( - "<|image_pad|>", - "<|vision_start|>", - "<|image_1|>", - "", - "", - ) - ) - - -def _load_vlm( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - workflow = str(options.get("workflow", "chat")) - if workflow == "document-ocr-infer": - return _load_deepseek_ocr(arguments, request) - if workflow == "grounded-generation": - return _load_locateanything(arguments, request) - if workflow not in {"chat", "single-image-token-chat"}: - raise ValueError(f"unsupported vision-language workflow: {workflow}") - - import torch - import transformers - from PIL import Image - from transformers import AutoProcessor - - device = torch.device("cuda") - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - load_options = _load_kwargs(arguments, torch) - if bool(options.get("eager_attention", False)): - config = transformers.AutoConfig.from_pretrained( - arguments.model, **_processor_kwargs(arguments) - ) - config._attn_implementation = "eager" - config._attn_implementation_internal = "eager" - load_options.update({"config": config, "attn_implementation": "eager"}) - model = ( - _load_vlm_model( - transformers, - str(options.get("model_class", "AutoModelForImageTextToText")), - arguments.model, - load_options, - ) - .eval() - .to(device) - ) - image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") - prompt = str(request.get("prompt", "")) - max_new_tokens = int(request.get("max_new_tokens", 16)) - - if workflow == "single-image-token-chat": - messages = [{"role": "user", "content": f"<|image_1|>{prompt}"}] - template_owner = getattr(processor, "tokenizer", processor) - else: - messages = [ - { - "role": "user", - "content": [ - {"type": "image", "image": image}, - {"type": "text", "text": prompt}, - ], - } - ] - template_owner = processor - rendered = template_owner.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True - ) - if not isinstance(rendered, str) or not _vl_prompt_has_image_placeholder(rendered): - raise ValueError("vision-language chat template lost the image placeholder") - inputs = processor(text=rendered, images=image, padding=True, return_tensors="pt") - inputs = _to_device(inputs, device, next(model.parameters()).dtype) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - output_ids = model.generate( - **inputs, - max_new_tokens=max_new_tokens, - do_sample=False, - num_beams=1, - ) - input_length = int(inputs["input_ids"].shape[-1]) - sequence = output_ids[0] - generated = sequence[input_length:] if sequence.shape[0] > input_length else sequence - token_ids = [int(token) for token in generated.detach().cpu().tolist()] - text = processor.decode(generated, skip_special_tokens=True) - return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} - - return Session(invoke, "transformers") - - -def _load_embedding( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - configured = _json_object( - getattr(arguments, "timing_contract_json", "{}"), "--timing-contract-json" - ) - if not configured: - raise ValueError("embedding reference requires explicit --timing-contract-json") - declared_timing = timing_contract(runner="task-reference", declared=configured) - if declared_timing["asset_loading_included"]: - raise ValueError( - "embedding reference preloads its assets; asset_loading_included must be false" - ) - prompts = _batch_prompt(request) - if len(prompts) != 1: - raise ValueError("embedding reference requires exactly one text input") - import torch - from transformers import AutoModel, AutoTokenizer - - device = torch.device("cuda") - tokenizer = AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - AutoModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - compile_evidence = _compile_forward(model) if arguments.mode == "torch-compile" else None - prompt = prompts[0] - - def prepare_inputs() -> Mapping[str, Any]: - return _to_device( - tokenizer(prompt, return_tensors="pt", truncation=True), - device, - ) - - prepared_inputs = None - if not declared_timing["input_preparation_included"]: - prepared_inputs = prepare_inputs() - - def invoke() -> Any: - inputs = prepare_inputs() if prepared_inputs is None else prepared_inputs - with torch.inference_mode(): - outputs = model(**inputs, output_hidden_states=True) - hidden = getattr(outputs, "last_hidden_state", None) - if hidden is None: - hidden = outputs.hidden_states[-1] - mask = inputs.get("attention_mask", torch.ones(hidden.shape[:2], device=device)) - mask = mask.unsqueeze(-1).to(hidden.dtype) - vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) - vector = torch.nn.functional.normalize(vector, p=2, dim=-1) - return vector.detach().to("cpu", dtype=torch.float32) - - def summarize(vector: Any) -> Mapping[str, Any]: - summary = _tensor_summary(vector) - summary.update( - { - "embedding_vectors": 1, - "embedding_elements": int(vector.numel()), - "dim": int(vector.shape[-1]), - } - ) - return summary - - return Session( - invoke, - "transformers", - timing_scope=str(declared_timing["timing_scope"]), - input_preparation_included=bool(declared_timing["input_preparation_included"]), - asset_loading_included=False, - compile_evidence=compile_evidence, - summarize=summarize, - ) - - -def _load_reranking( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - import torch - from transformers import AutoModelForSequenceClassification, AutoProcessor - - device = torch.device("cuda") - documents = [str(document) for document in request.get("documents", [])] - query = str(request.get("query", "")) - model = ( - AutoModelForSequenceClassification.from_pretrained( - arguments.model, **_load_kwargs(arguments, torch) - ) - .eval() - .to(device) - ) - processor = AutoProcessor.from_pretrained( - arguments.model, - max_input_tiles=6, - use_thumbnail=True, - rerank_max_length=8192, - **_processor_kwargs(arguments), - ) - examples = [ - {"question": query, "doc_text": document, "doc_image": ""} for document in documents - ] - inputs = processor.process_queries_documents_crossencoder(examples) - inputs = _to_device(inputs, device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - logits = model(**inputs).logits.detach().float().cpu() - if logits.ndim == 2: - logits = logits[:, 0] if logits.shape[-1] == 1 else logits[:, -1] - scores = [float(score) for score in logits.reshape(-1).tolist()] - return {"scores": scores, "document_count": len(documents)} - - return Session(invoke, "transformers") - - -_PIXART_TRTMC_MIXED_PRECISION = "pixart_fp16_dit_fp32_t5" - - -def _diffusion_component_precision_contract( - arguments: argparse.Namespace, - options: Mapping[str, Any], -) -> str: - contract = str(options.get("component_precision_contract", "") or "") - if not contract: - return "" - if contract != _PIXART_TRTMC_MIXED_PRECISION: - raise ValueError(f"unsupported diffusion component precision contract: {contract}") - if arguments.precision != "fp16": - raise ValueError(f"{contract} requires precision=fp16") - return contract - - -def _cast_floating_tensors(value: Any, dtype: Any, torch_module: Any) -> Any: - if isinstance(value, torch_module.Tensor): - return value.to(dtype=dtype) if value.is_floating_point() else value - if isinstance(value, tuple): - return tuple(_cast_floating_tensors(item, dtype, torch_module) for item in value) - if isinstance(value, list): - return [_cast_floating_tensors(item, dtype, torch_module) for item in value] - if isinstance(value, dict): - return { - key: _cast_floating_tensors(item, dtype, torch_module) for key, item in value.items() - } - return value - - -def _configure_diffusion_component_precision( - pipeline: Any, - arguments: argparse.Namespace, - options: Mapping[str, Any], - torch_module: Any, -) -> None: - contract = _diffusion_component_precision_contract(arguments, options) - if not contract: - return - - def fp16_transformer_inputs(_module: Any, args: Any, kwargs: Any) -> Any: - return ( - _cast_floating_tensors(args, torch_module.float16, torch_module), - _cast_floating_tensors(kwargs, torch_module.float16, torch_module), - ) - - def fp32_transformer_output(_module: Any, _args: Any, output: Any) -> Any: - return _cast_floating_tensors(output, torch_module.float32, torch_module) - - pipeline.transformer.register_forward_pre_hook(fp16_transformer_inputs, with_kwargs=True) - pipeline.transformer.register_forward_hook(fp32_transformer_output) - - -def _diffusion_pipeline( - arguments: argparse.Namespace, - torch_module: Any, - options: Mapping[str, Any], -) -> Any: - import diffusers - - model_id = str(options.get("model_id", arguments.model)) - requested_revision = ( - str(options.get("model_revision", getattr(arguments, "revision", None) or "")) or None - ) - model_source: str | Path = model_id - if arguments.local_files_only: - cached = _cached_snapshot_path(model_id, requested_revision, "model_index.json") - if cached is None: - raise FileNotFoundError(f"cached Diffusers snapshot is missing: {model_id}") - model_source = cached - - load_options: dict[str, Any] = {} - component_contract = _diffusion_component_precision_contract(arguments, options) - if component_contract == _PIXART_TRTMC_MIXED_PRECISION: - from transformers import T5EncoderModel - - load_options["text_encoder"] = T5EncoderModel.from_pretrained( - model_source, - subfolder="text_encoder", - torch_dtype=torch_module.float32, - **( - {"revision": requested_revision} - if requested_revision and model_source == model_id - else {} - ), - local_files_only=arguments.local_files_only, - ) - vae_class_name = options.get("vae_class") - if vae_class_name is not None: - if not isinstance(vae_class_name, str) or not vae_class_name.startswith("Autoencoder"): - raise ValueError("diffusion adapter_options.vae_class must name an Autoencoder class") - try: - vae_class = getattr(diffusers, vae_class_name) - except AttributeError as error: - raise ValueError(f"unsupported Diffusers VAE class: {vae_class_name}") from error - vae_precision = str(options.get("vae_precision", arguments.precision)) - load_options["vae"] = vae_class.from_pretrained( - model_source, - subfolder="vae", - torch_dtype=_torch_dtype(torch_module, vae_precision), - **( - {"revision": requested_revision} - if requested_revision and model_source == model_id - else {} - ), - local_files_only=arguments.local_files_only, - ) - return diffusers.DiffusionPipeline.from_pretrained( - model_source, - torch_dtype=_torch_dtype(torch_module, arguments.precision), - **load_options, - **( - {"revision": requested_revision} - if requested_revision and model_source == model_id - else {} - ), - trust_remote_code=bool(options.get("trust_remote_code", arguments.trust_remote_code)), - local_files_only=arguments.local_files_only, - ) - - -def _load_diffusers( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import inspect - import torch - from PIL import Image - - pipeline = _diffusion_pipeline(arguments, torch, options) - _configure_diffusion_component_precision(pipeline, arguments, options, torch) - if bool(options.get("cpu_offload", False)) and hasattr(pipeline, "enable_model_cpu_offload"): - pipeline.enable_model_cpu_offload() - else: - pipeline.to("cuda") - signature = inspect.signature(pipeline.__call__) - accepted = set(signature.parameters) - accepts_extra = any( - parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in signature.parameters.values() - ) - if "prompt" in request and "prompts" in request: - raise ValueError("reference request must not provide both prompt and prompts") - prompt = request.get("prompts", request.get("prompt", "")) - if "prompts" in request and not isinstance(prompt, list): - raise ValueError("prompts must contain one string per batch item") - if isinstance(prompt, list): - if not prompt or any(not isinstance(value, str) for value in prompt): - raise ValueError("prompt list must contain one string per batch item") - count = len(prompt) - else: - if not isinstance(prompt, str): - raise ValueError("prompt must be a string or a non-empty list of strings") - count = 1 - batch_size = request.get("batch_size", count) - if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - if isinstance(prompt, list): - if count != batch_size: - raise ValueError("prompt list must contain one string per batch item") - prompt_value: str | list[str] = list(prompt) - else: - prompt_value = [prompt] * batch_size if batch_size > 1 else prompt - values: dict[str, Any] = {"prompt": prompt_value} - negative_prompt = str(request.get("negative_prompt", "")) - if ( - negative_prompt - or "negative_prompt" in request - or bool(options.get("always_negative_prompt", False)) - ): - values["negative_prompt"] = negative_prompt - steps = int(request.get("num_steps", -1)) - if steps > 0: - values.update({"num_inference_steps": steps, "step": steps}) - manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) - height = int( - request.get("height", request.get("video_height", manifest.get("image_height", 0))) - ) - width = int(request.get("width", request.get("video_width", manifest.get("image_width", 0)))) - if height > 0: - values["height"] = height - if width > 0: - values["width"] = width - num_frames = int( - request.get( - "video_num_frames", - _task_value(arguments, request, "num_frames", manifest.get("video_num_frames", 1)), - ) - ) - if num_frames > 0: - values["num_frames"] = num_frames - for name in ("action", "intrinsics"): - value = str(_task_value(arguments, request, name, "") or "") - if value: - values[name] = value - fps = int(_task_value(arguments, request, "fps", 0)) - if fps > 0: - values["fps"] = fps - for name in ("flow_shift", "translation_speed", "rotation_speed_deg"): - value = _task_value(arguments, request, name) - if value is not None: - values[name] = float(value) - cfg_scale = float(request.get("cfg_scale", -1.0)) - guidance_parameter = str(options.get("guidance_parameter", "guidance_scale")) - if guidance_parameter not in {"guidance_scale", "true_cfg_scale"}: - raise ValueError(f"unsupported diffusion guidance_parameter: {guidance_parameter}") - if cfg_scale >= 0 and guidance_parameter != "true_cfg_scale": - values["cfg_scale"] = cfg_scale - if bool(request.get("no_refiner", False)): - values["no_refiner"] = True - guidance = float(request.get("guidance_scale", -1.0)) - if guidance_parameter == "true_cfg_scale": - true_cfg_scale = guidance if guidance >= 0 else cfg_scale - if true_cfg_scale >= 0: - values["true_cfg_scale"] = true_cfg_scale - elif guidance >= 0: - values["guidance_scale"] = guidance - values["output_type"] = "np" - if "image_path" in request and "image_paths" in request: - raise ValueError("reference request must not provide both image_path and image_paths") - image_path = request.get("image_path", "") - if "image_paths" in request: - image_paths = request["image_paths"] - if ( - not isinstance(image_paths, list) - or len(image_paths) != 1 - or not isinstance(image_paths[0], str) - or not image_paths[0] - ): - raise ValueError("this reference supports exactly one conditioning image") - image_path = image_paths[0] - task = getattr(arguments, "selected_task", None) or manifest["task"] - if task in {"images_text_to_image_edit", "image_edit"} and not image_path: - raise ValueError("image edit reference requires one conditioning image") - if image_path: - if "image" not in accepted and not accepts_extra: - raise ValueError("selected reference pipeline does not accept conditioning image") - values["image"] = Image.open( - _asset_path(arguments, {"image_path": image_path}, "image_path") - ).convert("RGB") - call_values = { - name: value for name, value in values.items() if name in accepted or accepts_extra - } - required = [str(name) for name in options.get("required_call_arguments", [])] - missing = [name for name in required if name not in call_values] - if missing: - raise ValueError( - f"{arguments.family} reference is missing required call arguments: " - + ", ".join(missing) - ) - seed = int(request.get("seed", 42)) - raw_seeds = request.get("seeds") - if raw_seeds is not None: - if not isinstance(raw_seeds, list) or len(raw_seeds) != batch_size: - raise ValueError("seeds must contain one integer per batch item") - seeds: int | list[int] = [int(value) for value in raw_seeds] - elif batch_size > 1: - seeds = [seed] * batch_size - else: - seeds = seed - if "generator" in accepted or accepts_extra: - if isinstance(seeds, list): - call_values["generator"] = [ - torch.Generator("cuda").manual_seed(value) for value in seeds - ] - else: - call_values["generator"] = torch.Generator("cuda").manual_seed(seeds) - - def invoke() -> Mapping[str, Any]: - if "generator" in call_values: - generators = call_values["generator"] - if isinstance(generators, list): - for generator, value in zip(generators, seeds, strict=True): - generator.manual_seed(value) - else: - generators.manual_seed(seeds) - result = pipeline(**call_values) - media = getattr(result, "images", None) - if media is None: - media = getattr(result, "frames", None) - media_type = str(request.get("media_type", "image")) - return {**_media_summary(media, media_type), "_media": media} - - return Session( - invoke, - "diffusers", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - - -def _load_modular_diffusers( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - """Run a ModularPipeline whose workflow is declared by the family profile.""" - import torch - from diffusers import ModularPipeline - - workflow = str(options.get("workflow", "")) - output_name = str(options.get("output", "videos")) - if not workflow or not output_name: - raise ValueError("modular Diffusers requires adapter_options.workflow and output") - load_options = { - "workflow": workflow, - "local_files_only": arguments.local_files_only, - } - if arguments.revision: - load_options["revision"] = arguments.revision - pipeline = ModularPipeline.from_pretrained(arguments.model, **load_options) - pipeline.load_components( - dtype=_torch_dtype(torch, arguments.precision), - pretrained_model_name_or_path=arguments.model, - local_files_only=arguments.local_files_only, - ) - pipeline = pipeline.to("cuda") - manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) - seed = int(request.get("seed", 42)) - generator = torch.Generator().manual_seed(seed) - values: dict[str, Any] = { - "prompt": str(request.get("prompt", "")), - "height": int(request.get("height", manifest.get("image_height", 0))), - "width": int(request.get("width", manifest.get("image_width", 0))), - "num_inference_steps": int(request.get("num_steps", 0)), - "generator": generator, - "output": output_name, - "output_type": "np", - } - frames = int(request.get("video_num_frames", manifest.get("video_num_frames", 1))) - if frames > 1: - values["num_frames"] = frames - for source, target in ( - ("negative_prompt", "negative_prompt"), - ("guidance_scale", "guidance_scale"), - ("cfg_scale", "cfg_scale"), - ): - if source in request: - values[target] = request[source] - - def invoke() -> Mapping[str, Any]: - generator.manual_seed(seed) - media = pipeline(**values) - media_type = "video" if frames > 1 else "image" - return {**_media_summary(media, media_type), "_media": media} - - return Session( - invoke, - "diffusers-modular", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - - -def _media_count(media: Any, media_type: str) -> int: - """Count image batches or video frames without coercing arrays to bool.""" - if media is None: - return 0 - try: - outer_count = len(media) - except TypeError: - return 1 - if outer_count == 0 or media_type != "video": - return outer_count - try: - return len(media[0]) - except TypeError: - return outer_count - - -def _media_summary(media: Any, media_type: str) -> dict[str, Any]: - """Describe materialized image/video geometry without copying its pixels.""" - count = _media_count(media, media_type) - item = media - try: - item = item[0] - if media_type == "video": - item = item[0] - except (IndexError, KeyError, TypeError): - pass - width = height = channels = None - size = getattr(item, "size", None) - bands = getattr(item, "getbands", None) - if isinstance(size, tuple) and len(size) == 2 and callable(bands): - width, height = (int(value) for value in size) - channels = len(bands()) - else: - shape = tuple(int(value) for value in getattr(item, "shape", ())) - if len(shape) >= 3: - if shape[-1] in {1, 3, 4}: - height, width, channels = shape[-3:] - elif shape[-3] in {1, 3, 4}: - channels, height, width = shape[-3:] - summary = { - "media_type": media_type, - "media_count": count, - "height": height, - "width": width, - "channels": channels, - } - try: - import numpy as np - - array = np.asarray(media) - if np.issubdtype(array.dtype, np.number): - finite = bool(np.isfinite(array).all()) - if not finite: - raise RuntimeError("reference returned non-finite media values") - summary["finite"] = True - except (TypeError, ValueError): - pass - return summary - - -def _media_items(media: Any, media_type: str) -> list[Any]: - """Return image items, or frames from the first generated video.""" - import numpy as np - - if media_type == "video": - if isinstance(media, np.ndarray): - values = media[0] if media.ndim == 5 else media - else: - values = media[0] if isinstance(media, (list, tuple)) and media else media - else: - values = media - if isinstance(values, np.ndarray): - if values.ndim == 3: - return [values] - return [values[index] for index in range(len(values))] - if isinstance(values, (list, tuple)): - return list(values) - return [values] - - -def _write_media_artifacts(summary: dict[str, Any], output: Path) -> None: - """Materialize deterministic sample frames after reference timing completes.""" - import numpy as np - from PIL import Image - - media = summary.pop("_media") - items = _media_items(media, str(summary.get("media_type", "image"))) - declared = int(summary.get("media_count", 0)) - if declared != len(items) or declared < 1: - raise RuntimeError("reference media count differs from its materialized output") - indices = sorted({0, declared // 2, declared - 1}) - directory = output.with_suffix(".media").resolve() - directory.mkdir(parents=True, exist_ok=True) - artifacts = [] - for index in indices: - item = items[index] - values = np.asarray(item) - scale = 255.0 if np.issubdtype(values.dtype, np.floating) else 1.0 - image = ( - item - if isinstance(item, Image.Image) - else Image.fromarray(np.clip(values * scale, 0, 255).astype(np.uint8)) - ) - path = directory / f"{index:06d}.png" - image.convert("RGB").save(path) - artifacts.append(str(path)) - summary["artifact_indices"] = indices - summary["frame_artifacts" if summary.get("media_type") == "video" else "image_artifacts"] = ( - artifacts - ) - - -def _numeric_values(request: Mapping[str, Any], key: str) -> list[float]: - raw = request.get(key) - if not isinstance(raw, list) or not raw: - raise ValueError(f"time-series reference requires non-empty request.{key}") - return [float(value) for value in raw] - - -def _observed_values(request: Mapping[str, Any], count: int) -> list[float]: - raw = request.get("observed_mask") - if raw is None or raw == []: - return [1.0] * count - values = _numeric_values(request, "observed_mask") - if len(values) != count: - raise ValueError("time-series observed_mask must match past_values") - return values - - -def _align(values: Sequence[float], length: int, fill: float) -> list[float]: - result = [fill] * length - count = min(len(values), length) - result[-count:] = values[-count:] - return result - - -def _patchtst_task(config: Any) -> str: - for attribute in ("patchtst_task", "task_type", "problem_type"): - value = str(getattr(config, attribute, "") or "").lower() - if "class" in value: - return "classification" - if "regress" in value: - return "regression" - if "forecast" in value or "predict" in value: - return "forecast" - - architectures = getattr(config, "architectures", []) or [] - if isinstance(architectures, str): - architectures = [architectures] - for architecture in architectures: - value = str(architecture).lower() - if "class" in value: - return "classification" - if "regress" in value: - return "regression" - if "forecast" in value or "predict" in value: - return "forecast" - return "regression" - - -def _load_timeseries( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import torch - import transformers - - device = torch.device("cuda") - dtype = _torch_dtype(torch, arguments.precision) - task_id = _selected_task(arguments) - reference_type = str(options.get("reference_type", "")) - if reference_type == "chronos-bolt": - from chronos import ChronosBoltPipeline - - chronos_options = _processor_kwargs(arguments) - chronos_options.update({"device_map": str(device), "dtype": dtype}) - model = ChronosBoltPipeline.from_pretrained(arguments.model, **chronos_options) - compile_evidence = None - if arguments.mode == "torch-compile": - compile_evidence = _compile_forward(model.model) - raw = _numeric_values(request, "past_values") - observed = _observed_values(request, len(raw)) - context = torch.tensor( - [value if mask > 0 else float("nan") for value, mask in zip(raw, observed)], - dtype=dtype, - device=device, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - value = model.predict( - context, - prediction_length=model.model_prediction_length, - limit_prediction_length=True, - ) - quantiles = model.model.config.chronos_config["quantiles"] - return _forecast_summary(value, task_id, quantiles) - - return Session(invoke, "chronos", compile_evidence=compile_evidence) - - config = transformers.AutoConfig.from_pretrained( - arguments.model, **_processor_kwargs(arguments) - ) - if reference_type == "timesfm": - model = ( - transformers.TimesFmModelForPrediction.from_pretrained( - arguments.model, **_load_kwargs(arguments, torch) - ) - .eval() - .to(device) - ) - length = int(model.config.context_length) - raw = _numeric_values(request, "past_values") - series = torch.tensor(_align(raw, length, 0.0), dtype=dtype, device=device).reshape( - 1, length - ) - observed = _align(_observed_values(request, len(raw)), length, 0.0) - padding = [0 if mask > 0 else 1 for mask in observed] - padding_tensor = torch.tensor(padding, dtype=torch.int32, device=device).reshape(1, length) - frequency = int(request.get("frequency", 0)) - frequency_tensor = torch.tensor([[frequency]], dtype=torch.long, device=device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - decoder = model.decoder( - past_values=series, - past_values_padding=padding_tensor, - freq=frequency_tensor, - output_attentions=False, - output_hidden_states=False, - ) - output = model._postprocess_output( - decoder.last_hidden_state, (decoder.loc, decoder.scale) - )[:, -1, : model.config.horizon_length, 0] - return _forecast_summary(output, task_id) - - else: - if reference_type not in {"patchtsmixer", "patchtst"}: - raise ValueError(f"unsupported time-series reference_type: {reference_type}") - is_mixer = reference_type == "patchtsmixer" - if is_mixer: - model_class = transformers.PatchTSMixerForPrediction - output_name = "prediction_outputs" - else: - task = _patchtst_task(config) - class_name, output_name = { - "classification": ( - "PatchTSTForClassification", - "prediction_logits", - ), - "forecast": ("PatchTSTForPrediction", "prediction_outputs"), - "regression": ("PatchTSTForRegression", "regression_outputs"), - }[task] - model_class = getattr(transformers, class_name) - if not hasattr(model_class, "all_tied_weights_keys"): - model_class.all_tied_weights_keys = {} - model = ( - model_class.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - length = int(config.context_length) - channels = int(config.num_input_channels) - raw = _numeric_values(request, "past_values") - values = torch.tensor( - _align(raw, length * channels, 0.0), dtype=dtype, device=device - ).reshape(1, length, channels) - aligned_mask = _align(_observed_values(request, len(raw)), length * channels, 0.0) - observed = torch.tensor( - aligned_mask, - dtype=dtype, - device=device, - ).reshape(1, length, channels) - - def invoke() -> Mapping[str, Any]: - with ( - torch.inference_mode(), - torch.autocast( - device_type="cuda", - dtype=dtype, - enabled=dtype != torch.float32, - ), - ): - if is_mixer: - outputs = model( - past_values=values * observed, - observed_mask=observed, - return_loss=False, - return_dict=True, - ) - output = outputs.prediction_outputs - else: - outputs = model( - past_values=values, - past_observed_mask=observed.gt(0.5), - return_dict=True, - ) - output = getattr(outputs, output_name) - if task_id == "series_to_regression_distribution": - return _regression_summary( - output, config.distribution_output, tuple(model.distribution_output.args_dim) - ) - if task_id == "series_to_regression_values": - return _regression_values_summary(output) - if isinstance(output, (tuple, list)): - output = torch.stack(list(output), dim=-1) - return _forecast_summary(output, task_id) - - return Session(invoke, "transformers") - - -def _load_vision( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import torch - from PIL import Image - import transformers - - device = torch.device("cuda") - image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") - width, height = image.size - kwargs = _load_kwargs(arguments, torch) - processor_kwargs = _processor_kwargs(arguments) - vision_task = str(options.get("vision_task", "")) - - if arguments.adapter == "timm-classification": - import timm - from timm.data import create_transform, resolve_model_data_config - - model_id = arguments.model - if arguments.revision: - model_id += "@" + arguments.revision - model = timm.create_model(f"hf-hub:{model_id}", pretrained=True) - dtype = _torch_dtype(torch, arguments.precision) - model = model.eval().to(device=device, dtype=dtype) - transform = create_transform(**resolve_model_data_config(model), is_training=False) - inputs = transform(image).unsqueeze(0).to(device=device, dtype=dtype) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - logits = model(inputs) - return {"top_class": int(logits.argmax(dim=-1)[0]), **_tensor_summary(logits)} - - elif vision_task == "object-detection": - processor = transformers.AutoImageProcessor.from_pretrained( - arguments.model, **processor_kwargs - ) - model = ( - transformers.AutoModelForObjectDetection.from_pretrained(arguments.model, **kwargs) - .eval() - .to(device) - ) - inputs = _to_device( - processor(images=image, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - target_sizes = torch.tensor([[height, width]], device=device) - results = processor.post_process_object_detection( - outputs, threshold=0.5, target_sizes=target_sizes - )[0] - return { - "detected_images": 1, - "detections": int(results["scores"].shape[0]), - "image_height": height, - "image_width": width, - "boxes": results["boxes"].float().cpu().reshape(-1).tolist(), - "scores": results["scores"].float().cpu().tolist(), - "class_ids": results["labels"].cpu().tolist(), - "shape": [int(results["scores"].shape[0]), 4], - "coordinates": "xyxy", - "units": "pixels", - } - - elif vision_task == "image-features": - processor = transformers.AutoImageProcessor.from_pretrained( - arguments.model, **processor_kwargs - ) - model = transformers.AutoModel.from_pretrained(arguments.model, **kwargs).eval().to(device) - inputs = _to_device( - processor(images=image, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - return { - "last_hidden_state_shape": _tensor_summary(outputs.last_hidden_state)["shape"], - "pooler_output_shape": _tensor_summary(outputs.pooler_output)["shape"], - } - - elif vision_task == "semantic-segmentation": - processor = transformers.AutoImageProcessor.from_pretrained( - arguments.model, **processor_kwargs - ) - model = ( - transformers.AutoModelForSemanticSegmentation.from_pretrained(arguments.model, **kwargs) - .eval() - .to(device) - ) - inputs = _to_device( - processor(images=image, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - mask = processor.post_process_semantic_segmentation( - outputs, target_sizes=[(height, width)] - )[0] - return { - "num_masks": 1, - "height": int(mask.shape[0]), - "width": int(mask.shape[1]), - } - - elif vision_task == "text-prompted-segmentation": - processor = _load_sam3_processor(transformers, arguments.model, processor_kwargs) - model = transformers.Sam3Model.from_pretrained(arguments.model, **kwargs).eval().to(device) - inputs = _to_device( - processor( - images=image, - text=str(request.get("prompt", "")), - return_tensors="pt", - ), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - original_sizes = inputs.get("original_sizes") - target_sizes = ( - original_sizes.cpu().tolist() if hasattr(original_sizes, "cpu") else original_sizes - ) - result = processor.post_process_instance_segmentation( - outputs, - threshold=0.5, - mask_threshold=0.5, - target_sizes=target_sizes, - )[0] - masks = result.get("masks") - num_masks = 0 if masks is None else int(masks.shape[0] if masks.ndim > 2 else 1) - return { - "num_masks": num_masks, - "height": height, - "width": width, - } - - elif vision_task == "point-prompted-segmentation": - processor = transformers.SamProcessor.from_pretrained(arguments.model, **processor_kwargs) - model = transformers.SamModel.from_pretrained(arguments.model, **kwargs).eval().to(device) - points = [ - [ - [ - int(float(request.get("point_x", 0.5)) * width), - int(float(request.get("point_y", 0.5)) * height), - ] - ] - ] - inputs = _to_device( - processor(image, input_points=points, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - masks = processor.image_processor.post_process_masks( - outputs.pred_masks.cpu(), - inputs["original_sizes"].cpu(), - inputs["reshaped_input_sizes"].cpu(), - )[0] - height, width = (int(value) for value in masks.shape[-2:]) - num_masks = int(masks.numel() // (height * width)) - return { - "num_masks": num_masks, - "height": height, - "width": width, - } - - else: - raise ValueError(f"unsupported Transformers vision_task: {vision_task}") - - framework = "timm" if arguments.adapter == "timm-classification" else "transformers" - return Session(invoke, framework) - - -def _qwen3_omni_chat_inputs(processor: Any, conversation: Sequence[Mapping[str, Any]]) -> Any: - processor_template = getattr(processor, "chat_template", None) - tokenizer_template = getattr(getattr(processor, "tokenizer", None), "chat_template", None) - template = ( - processor_template - if isinstance(processor_template, str) and processor_template.strip() - else tokenizer_template - if isinstance(tokenizer_template, str) and tokenizer_template.strip() - else QWEN3_OMNI_TEXT_CHAT_TEMPLATE - ) - return processor.apply_chat_template( - conversation, - chat_template=template, - add_generation_prompt=True, - tokenize=True, - return_dict=True, - return_tensors="pt", - padding=True, - ) - - -def _load_qwen3_omni( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import torch - from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor - - processor = Qwen3OmniMoeProcessor.from_pretrained( - arguments.model, **_processor_kwargs(arguments) - ) - load_options = _load_kwargs(arguments, torch) - load_options.update( - { - "device_map": str(options.get("device_map", "cuda:0")), - "enable_audio_output": False, - } - ) - model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( - arguments.model, **load_options - ).eval() - conversation = [ - {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT_QWEN3_OMNI}]}, - {"role": "user", "content": [{"type": "text", "text": str(request.get("prompt", ""))}]}, - ] - inputs = _qwen3_omni_chat_inputs(processor, conversation).to(model.device) - thinker_tokens = int(request.get("max_new_tokens", 16)) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - text_ids = model.generate( - **inputs, - thinker_max_new_tokens=thinker_tokens, - thinker_do_sample=False, - return_audio=False, - ) - input_length = int(inputs["input_ids"].shape[-1]) - generated = text_ids[:, input_length:] - token_ids = [int(token) for token in generated[0].detach().cpu().tolist()] - return { - "text": processor.batch_decode(generated, skip_special_tokens=True)[0].strip(), - "token_ids": token_ids, - "output_tokens": len(token_ids), - } - - return Session(invoke, "transformers") - - -def _load_voicechat( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - """One loaded official offline pipeline, not a prerecorded or live-session oracle.""" - import soundfile as sf - import torch - - request = flatten_config(request) - task = _selected_task(arguments) - if arguments.operation == "speech_dialogue": - if task != "offline_speech_dialogue": - raise ValueError("offline VoiceChat reference cannot qualify live or tool dialogue") - elif arguments.operation == "speak": - if task not in { - "speech_to_speech", - "speech_to_speech_response", - "speech_session", - }: - raise ValueError("VoiceChat speak reference requires a speech-response contract") - else: - raise ValueError("VoiceChat reference supports speak or offline speech_dialogue") - if arguments.precision != "fp32": - raise ValueError("VoiceChat reference currently matches the fp32 native profile") - allowed = { - "audio_path", - "system_prompt", - "seed", - "temperature", - "top_p", - "repetition_penalty", - "presence_penalty", - "tail_frames", - "finish_tail_frames", - } - if unknown := sorted(request.keys() - allowed): - raise ValueError("unsupported offline VoiceChat request fields: " + ", ".join(unknown)) - for key in ("tail_frames", "finish_tail_frames"): - if key in request and ( - isinstance(request[key], bool) or not isinstance(request[key], int) or request[key] != 0 - ): - raise ValueError("offline VoiceChat reference requires zero explicit tail frames") - prompt = request.get("system_prompt", VOICECHAT_SYSTEM_PROMPT) - if not isinstance(prompt, str): - raise ValueError("VoiceChat system_prompt must be text") - # The native empty prompt selects its default, unlike upstream's blank-prompt helper. - if not prompt: - prompt = VOICECHAT_SYSTEM_PROMPT - elif not prompt.strip(): - raise ValueError("whitespace-only prompts do not have matched native/reference semantics") - seed = _request_seed(request, default=0) - if seed < 0: - raise ValueError("VoiceChat reference seed must be nonnegative") - generation = {} - for key in ("temperature", "top_p", "repetition_penalty", "presence_penalty"): - if key in request: - value = request[key] - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - ): - raise ValueError(f"VoiceChat {key} must be finite numeric data") - generation[key] = value - if generation.get("temperature", 0) < 0 or not 0 < generation.get("top_p", 1) <= 1: - raise ValueError("VoiceChat temperature/top_p is outside the supported range") - if generation.get("repetition_penalty", 1) <= 0: - raise ValueError("VoiceChat repetition_penalty must be positive") - - configured_repo = str(options.get("reference_repo", "")) - if not configured_repo: - raise ValueError("nemo-voicechat requires baseline.adapter_options.reference_repo") - reference_repo = Path(_reference_checkout(configured_repo, repository="NeMo Speech")) - utility = reference_repo / "nemo/collections/speechlm2/inference/utils/offline_voicechat.py" - if not utility.is_file(): - raise ValueError("VoiceChat reference checkout lacks the official offline utility") - sys.path.insert(0, str(reference_repo)) - upstream = importlib.import_module( - "nemo.collections.speechlm2.inference.utils.offline_voicechat" - ) - if Path(upstream.__file__).resolve() != utility.resolve(): - raise RuntimeError( - "VoiceChat imported a different NeMo installation; use the requested checkout first" - ) - - checkpoint = _cached_snapshot_path(arguments.model, arguments.revision, "config.json") - if checkpoint is None: - from huggingface_hub import snapshot_download - - checkpoint = Path( - snapshot_download( - repo_id=arguments.model, - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - ) - audio_path = _asset_path(arguments, request, "audio_path") - info = sf.info(audio_path) - # Loading is outside the clock. Reject inputs that would make the official - # WAV helper do otherwise-unmeasured resampling/downmixing there. - if info.samplerate != 16000 or info.channels != 1 or info.frames <= 0: - raise ValueError("matched offline VoiceChat reference requires nonempty 16 kHz mono audio") - model = upstream.build_model(str(checkpoint), device="cuda").float() - if model.source_sample_rate != 16000 or model.target_sample_rate != 22050: - raise ValueError("VoiceChat checkpoint sample rates differ from the matched native profile") - _waveform, signal, lengths = upstream.load_wav_16k_mono(str(audio_path), device="cpu") - - def invoke() -> Mapping[str, Any]: - _seed_all(torch, seed) - # Host-to-device input transfer and prompt preparation belong to the - # public operation, while checkpoint loading and WAV decoding do not. - input_signal, input_lengths = signal.to("cuda"), lengths.to("cuda") - prompt_tokens, prompt_lengths = upstream.encode_system_prompt(model, prompt, device="cuda") - result = upstream.run_offline_inference( - model, - input_signal=input_signal, - input_signal_lens=input_lengths, - prompt_tokens=prompt_tokens, - prompt_token_lens=prompt_lengths, - decode_audio=True, - input_pad_len=0, - **generation, - ) - audio = result.get("audio") - audio_len = result.get("audio_len") - text = result.get("text") - if ( - audio is None - or audio_len is None - or len(audio.shape) != 2 - or audio.shape[0] != 1 - or tuple(audio_len.shape) != (1,) - or not isinstance(text, list) - or len(text) != 1 - or not isinstance(text[0], str) - ): - raise RuntimeError("VoiceChat returned an invalid one-recording audio/text result") - count = audio_len[0].item() - if ( - isinstance(count, bool) - or not isinstance(count, int) - or count < 0 - or count > audio.shape[1] - ): - raise RuntimeError("VoiceChat audio_len does not match the decoded audio") - materialized = audio[0, :count].detach().float().cpu() - if not bool(materialized.isfinite().all().item()): - raise RuntimeError("VoiceChat reference returned nonfinite audio") - return { - "text": text[0], - "audio_samples": count, - "num_samples": count, - "sample_rate": int(model.target_sample_rate), - "channels": 1, - "audio_seconds": count / model.target_sample_rate, - "input_sample_rate": 16000, - "input_channels": 1, - "input_samples": int(info.frames), - "finite": True, - "_audio_f32": materialized.numpy(), - } - - return Session( - invoke, - "nemo-voicechat", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=False, - ) - - -def _load_moge( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import numpy as np - import torch - from PIL import Image - from torch.nn.attention import SDPBackend, sdpa_kernel - - request = flatten_config(request) - if arguments.operation != "geometry" or arguments.precision != "fp32": - raise ValueError("MoGe reference requires geometry with the qualified fp32 profile") - if unknown := sorted(request.keys() - {"image_path", "num_tokens", "fov_x"}): - raise ValueError("unsupported MoGe reference request fields: " + ", ".join(unknown)) - num_tokens = request.get("num_tokens", options.get("num_tokens")) - if isinstance(num_tokens, bool) or not isinstance(num_tokens, int) or num_tokens != 1800: - raise ValueError("MoGe reference requires explicit request/options num_tokens=1800") - if ( - "num_tokens" in request - and "num_tokens" in options - and request["num_tokens"] != options["num_tokens"] - ): - raise ValueError("MoGe request and adapter num_tokens disagree") - inference = { - "num_tokens": num_tokens, - "use_fp16": False, - "force_projection": True, - "apply_mask": True, - } - if "fov_x" in request: - fov = request["fov_x"] - if ( - isinstance(fov, bool) - or not isinstance(fov, (int, float)) - or not math.isfinite(fov) - or not 0 < fov < 180 - ): - raise ValueError("MoGe fov_x must be a finite angle in degrees between zero and 180") - inference["fov_x"] = fov - configured_repo = str(options.get("reference_repo", "")) - if not configured_repo: - raise ValueError("upstream-moge requires baseline.adapter_options.reference_repo") - reference_repo = Path(_reference_checkout(configured_repo, repository="microsoft/MoGe")) - expected_module = reference_repo / "moge/model/v2.py" - if not expected_module.is_file(): - raise ValueError("MoGe reference checkout does not contain the v2 model") - # The upstream checkout supplies its own declared geometry dependencies. - sys.path.insert(0, str(reference_repo)) - upstream = importlib.import_module("moge.model.v2") - if Path(upstream.__file__).resolve() != expected_module.resolve(): - raise RuntimeError("MoGe imported a different installation than the requested checkout") - cached = _cached_snapshot_path(arguments.model, arguments.revision, "model.pt") - if cached is None: - from huggingface_hub import hf_hub_download - - checkpoint_path = Path( - hf_hub_download( - repo_id=arguments.model, - revision=arguments.revision, - filename="model.pt", - local_files_only=arguments.local_files_only, - ) - ) - else: - checkpoint_path = cached if cached.is_file() else cached / "model.pt" - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True, mmap=True) - if set(checkpoint) != {"model_config", "model"}: - raise ValueError("MoGe checkpoint has an unexpected top-level contract") - model = upstream.MoGeModel(**checkpoint["model_config"]) - missing, unexpected = model.load_state_dict(checkpoint["model"], strict=False) - if missing or unexpected: - raise ValueError(f"MoGe state mismatch: missing={missing}, unexpected={unexpected}") - del checkpoint - model.onnx_compatible_mode = True - model.eval().float().to("cuda") - torch.backends.cuda.matmul.allow_tf32 = False - torch.backends.cudnn.allow_tf32 = False - image_path = _asset_path(arguments, request, "image_path") - with Image.open(image_path) as image: - # Match the native caller's cached RGB float HWC input. No resizing or - # model normalization is performed here; upstream infer owns those. - pixels = np.asarray(image.convert("RGB"), dtype=np.float32).copy() / 255.0 - height, width = pixels.shape[:2] - - def invoke() -> Mapping[str, Any]: - tensor = torch.from_numpy(pixels).permute(2, 0, 1).to("cuda") - with sdpa_kernel([SDPBackend.MATH]): - output = model.infer(tensor, **inference) - if set(output) != {"points", "depth", "mask", "intrinsics"}: - raise RuntimeError("MoGe reference must return points, depth, mask and intrinsics") - arrays = {name: value.detach().cpu().numpy() for name, value in output.items()} - return { - "geometry_images": 1, - "geometry_pixels": height * width, - "height": height, - "width": width, - "point_shape": [height, width, 3], - "num_tokens": num_tokens, - "requested_fov_x": request.get("fov_x"), - "units": "meters", - "camera_axes": ["right", "down", "forward"], - "intrinsics_coordinates": "normalized_uv", - "_geometry_arrays": arrays, - } - - return Session( - invoke, - "moge-pytorch", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=False, - ) - - -def _write_geometry_artifacts(summary: dict[str, Any], output: Path) -> None: - """Validate and write complete geometry only after the measurement clock.""" - import numpy as np - - arrays = summary.pop("_geometry_arrays") - height, width = summary["height"], summary["width"] - points, depth, mask, intrinsics = ( - np.asarray(arrays[name]) for name in ("points", "depth", "mask", "intrinsics") - ) - if ( - points.shape != (height, width, 3) - or depth.shape != (height, width) - or mask.shape != (height, width) - or intrinsics.shape != (3, 3) - or not np.isin(mask, (0, 1)).all() - or not np.isfinite(intrinsics).all() - ): - raise RuntimeError("MoGe geometry arrays do not match the original image/calibration") - valid = mask.astype(bool, copy=False) - if ( - not np.isfinite(points[valid]).all() - or not np.isfinite(depth[valid]).all() - or np.any(depth[valid] <= 0) - or not np.isposinf(points[~valid]).all() - or not np.isposinf(depth[~valid]).all() - ): - raise RuntimeError( - "MoGe geometry must preserve valid metric values and masked positive infinity" - ) - prefix = output.with_suffix(".geometry").resolve() - prefix.parent.mkdir(parents=True, exist_ok=True) - for name, values, suffix, dtype in ( - ("points_artifact", points, ".points.f32", " Session: - official_repo = str(options.get("official_repo", "")) - if not official_repo: - raise ValueError("pytorch-personaplex requires baseline.adapter_options.official_repo") - official_repo = _reference_checkout( - official_repo, repository="https://github.com/NVIDIA/personaplex" - ) - import importlib.util - - if importlib.util.find_spec("sphn") is None: - raise RuntimeError("PersonaPlex reference requires the official sphn package") - sys.path[:0] = [str(Path(official_repo) / "moshi"), official_repo] - import torch - from huggingface_hub import hf_hub_download - from moshi.models import LMGen, loaders - from moshi.models.lm import _iterate_audio, encode_from_sphn, load_audio - from moshi.offline import warmup - - device = "cuda" - mimi_weights = hf_hub_download( - arguments.model, - loaders.MIMI_NAME, - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - model_weights = hf_hub_download( - arguments.model, - loaders.MOSHI_NAME, - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - mimi = loaders.get_mimi(mimi_weights, device) - other_mimi = loaders.get_mimi(mimi_weights, device) - language_model = loaders.get_moshi_lm(model_weights, device=device).eval() - frame_size = int(mimi.sample_rate / mimi.frame_rate) - generator = LMGen( - language_model, - audio_silence_frame_cnt=0, - sample_rate=mimi.sample_rate, - device=device, - frame_rate=mimi.frame_rate, - use_sampling=False, - temp=0.8, - temp_text=0.7, - top_k=250, - top_k_text=25, - ) - mimi.streaming_forever(1) - other_mimi.streaming_forever(1) - generator.streaming_forever(1) - warmup(mimi, other_mimi, generator, device, frame_size) - source_audio = load_audio(str(_asset_path(arguments, request, "audio_path")), mimi.sample_rate) - max_frames = int(request.get("max_new_tokens", options.get("max_frames", 100))) - - def invoke() -> Mapping[str, Any]: - mimi.reset_streaming() - other_mimi.reset_streaming() - generator.reset_streaming() - generated_frames = 0 - decoded_chunks = [] - - def summary() -> Mapping[str, Any]: - audio = ( - torch.cat(decoded_chunks, dim=-1).detach().float().cpu().reshape(-1).numpy() - if decoded_chunks - else torch.empty(0, dtype=torch.float32).numpy() - ) - return { - "audio_frames": generated_frames, - "audio_samples": int(audio.size), - "sample_rate": mimi.sample_rate, - "_audio_f32": audio, - } - - with torch.inference_mode(): - for encoded in encode_from_sphn( - mimi, - _iterate_audio(source_audio, sample_interval_size=frame_size, pad=True), - max_batch=1, - ): - for index in range(encoded.shape[-1]): - tokens = generator.step(encoded[:, :, index : index + 1]) - if tokens is None: - continue - decoded_chunks.append(mimi.decode(tokens[:, 1:9])) - other_mimi.decode(tokens[:, 1:9]) - generated_frames += 1 - if generated_frames >= max_frames: - return summary() - return summary() - - return Session( - invoke, - "moshi", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - - -def _load_fast_foundation_stereo( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import numpy as np - import torch - from PIL import Image - - from apps.benchmark.performance.baselines.reference_support import ( - configure_official_model_args, - fast_foundation_stereo_model_dir, - ) - - if not torch.cuda.is_available(): - raise RuntimeError("CUDA is required for the Fast Foundation Stereo reference") - max_disp = int(request.get("max_disp", 192)) - valid_iters = int(request.get("valid_iters", 8)) - height = int(request.get("height", 700)) - width = int(request.get("width", 700)) - if (height, width, max_disp, valid_iters) != (700, 700, 192, 8): - raise ValueError( - "Fast Foundation Stereo performance requires 700x700, max_disp=192, and valid_iters=8" - ) - model_dir = options.get("model_dir") - if not isinstance(model_dir, str) or not model_dir: - raise ValueError("Fast Foundation Stereo reference requires adapter_options.model_dir") - model_root = fast_foundation_stereo_model_dir(model_dir) - previous_cwd = Path.cwd() - try: - os.chdir(model_root) - sys.path.insert(0, str(model_root)) - from core.utils.utils import InputPadder - from Utils import AMP_DTYPE - - checkpoint = model_root / "weights/23-36-37/model_best_bp2_serialize.pth" - model = torch.load(checkpoint, map_location="cpu", weights_only=False) - finally: - os.chdir(previous_cwd) - configure_official_model_args( - model, - max_disparity=max_disp, - valid_iters=valid_iters, - ) - model = model.cuda().eval() - - def load_image(key: str) -> np.ndarray: - with Image.open(_asset_path(arguments, request, key)) as image: - pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) - if pixels.shape != (height, width, 3): - raise ValueError(f"{key} must be 700x700 RGB, got {pixels.shape}") - return pixels - - left = load_image("left_image_path") - right = load_image("right_image_path") - - def invoke() -> Mapping[str, Any]: - left_tensor = torch.as_tensor(left).cuda().float()[None].permute(0, 3, 1, 2) - right_tensor = torch.as_tensor(right).cuda().float()[None].permute(0, 3, 1, 2) - padder = InputPadder(left_tensor.shape, divis_by=32, force_square=False) - left_tensor, right_tensor = padder.pad(left_tensor, right_tensor) - with ( - torch.inference_mode(), - torch.amp.autocast("cuda", enabled=True, dtype=AMP_DTYPE), - ): - disparity = model.forward( - left_tensor, - right_tensor, - iters=valid_iters, - test_mode=True, - optimize_build_volume="pytorch1", - ) - output = padder.unpad(disparity.float()).cpu().numpy().reshape(height, width) - output = np.clip(output, 0, None).astype(np.float32, copy=False) - return { - "height": height, - "width": width, - "element_count": int(output.size), - "finite_fraction": float(np.isfinite(output).mean()), - "nonnegative_fraction": float((output >= 0).mean()), - "_disparity_f32": output, - } - - return Session( - invoke, - f"torch-{torch.__version__}", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=False, - ) - - -LOADERS: dict[ - str, Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session] -] = { - "hf-diffusers": _load_diffusers, - "hf-diffusers-modular": _load_modular_diffusers, - "hf-chat-asr": _load_asr, - "hf-qwen3-omni": _load_qwen3_omni, - "hf-transformers-asr": _load_asr, - "hf-transformers-embedding": _load_embedding, - "hf-transformers-reranking": _load_reranking, - "hf-transformers-tts": _load_tts, - "hf-transformers-vision": _load_vision, - "hf-transformers-vlm": _load_vlm, - "nemo-asr": _load_asr, - "nemo-tts": _load_tts, - "nemo-voicechat": _load_voicechat, - "upstream-moge": _load_moge, - "pytorch-personaplex": _load_personaplex, - "pytorch-timeseries": _load_timeseries, - "timm-classification": _load_vision, - "upstream-fast-foundation-stereo": _load_fast_foundation_stereo, -} - - -def _synchronize() -> None: - try: - import torch - - if torch.cuda.is_available(): - torch.cuda.synchronize() - except ImportError: - return - - -def _compile_forward(model: Any) -> dict[str, Any]: - import torch - from torch._dynamo.backends.registry import lookup_backend - - evidence = {"compiled_graph_count": 0} - inductor = lookup_backend("inductor") - - def compile_graph(graph: Any, inputs: Any, **options: Any) -> Any: - compiled = inductor(graph, inputs, **options) - evidence["compiled_graph_count"] += 1 - return compiled - - model.forward = torch.compile( - model.forward, - backend=compile_graph, - fullgraph=False, - dynamic=False, - ) - evidence.update( - { - "api": "torch.compile", - "target": "model.forward", - "backend": "inductor", - "mode": "default", - "fullgraph": False, - "dynamic": False, - "applied": True, - } - ) - return evidence - - -def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: - output: Any = {} - untimed_iterations = max(warmup, int(session.compile_evidence is not None)) - for _ in range(untimed_iterations): - output = session.invoke() - _synchronize() - compiled_graphs = None - if session.compile_evidence is not None: - compiled_graphs = int(session.compile_evidence["compiled_graph_count"]) - if compiled_graphs < 1: - raise RuntimeError("warmup did not execute a compiled graph") - samples = [] - for _ in range(iterations): - _synchronize() - started = time.perf_counter() - output = session.invoke() - _synchronize() - samples.append((time.perf_counter() - started) * 1000.0) - if ( - compiled_graphs is not None - and int(session.compile_evidence["compiled_graph_count"]) != compiled_graphs - ): - raise RuntimeError("model compilation occurred inside timed samples") - summary = session.summarize(output) if session.summarize is not None else output - return samples, dict(summary) - - -def _run_elf( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - reference_repo = str(options.get("reference_repo", "")) - if not reference_repo: - raise ValueError("upstream-elf requires baseline.adapter_options.reference_repo") - reference_repo = _reference_checkout( - reference_repo, repository="https://github.com/lillian039/ELF" - ) - count = arguments.warmup + arguments.iterations - prompt = str(request.get("prompt", "")) - - def repository_input(name: str) -> str: - value = str(request.get(name, "") or "") - if not value: - return "" - path = Path(value) - model_root = arguments.manifest.resolve().parent.parent - return str(path if path.is_absolute() else (model_root / path).resolve()) - - with tempfile.TemporaryDirectory(prefix="trtmc-perf-elf-") as temporary: - root = Path(temporary) - dataset = root / "dataset.jsonl" - rows = [ - {"id": f"perf_{index:04d}", "input": prompt, "output": ""} for index in range(count) - ] - dataset.write_text( - "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), - encoding="utf-8", - ) - output = root / "predictions.json" - command = [ - sys.executable, - str(REPOSITORY / "apps/benchmark/performance/baselines/elf_reference.py"), - "--reference-repo", - reference_repo, - "--config", - str(options["config"]), - "--checkpoint", - str(options["checkpoint"]), - "--dataset", - str(dataset), - "--output", - str(output), - "--shared-inputs-dir", - str(root / "shared"), - "--generation-mode", - str(request.get("generation_mode", "conditional")), - "--sampling-method", - str(options.get("sampling_method", "ode")), - "--num-steps", - str(request.get("num_steps", 64)), - "--cfg-scale", - str(request.get("cfg_scale", 2.0)), - "--self-cond-cfg-scale", - str(options.get("self_cond_cfg_scale", 1.0)), - "--sde-gamma", - str(options.get("sde_gamma", 0.0)), - "--seed", - str(options.get("seed", 42)), - "--precision", - arguments.precision, - ] - replay_inputs = ( - ("initial_latents_path", "--initial-latents"), - ("sampling_steps_path", "--sampling-steps"), - ("condition_latents_path", "--condition-latents"), - ("condition_mask_path", "--condition-mask"), - ("sde_noises_path", "--sde-noises"), - ) - for request_name, flag in replay_inputs: - value = repository_input(request_name) - if value: - command.extend([flag, value]) - if arguments.local_files_only: - command.append("--local-files-only") - completed = subprocess.run( - command, - cwd=reference_repo, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - f"ELF reference failed with rc={completed.returncode}: {completed.stderr[-2000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - responses = payload.get("responses", []) - if len(responses) != count: - raise RuntimeError(f"ELF reference returned {len(responses)} responses; expected {count}") - samples = [float(row["wall_ms"]) for row in responses[arguments.warmup :]] - summary_row = responses[arguments.warmup] - summary = { - "text": str(summary_row.get("output_text", "")), - "token_ids": summary_row.get("generated_token_ids", []), - "output_tokens": len(summary_row.get("generated_token_ids", [])), - } - return ( - samples, - summary, - "elf-pytorch", - "task-model-call-wall", - False, - False, - ) - - -def _run_lance( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - reference_repo = str(options.get("reference_repo", "")) - if not reference_repo: - raise ValueError("upstream-lance requires baseline.adapter_options.reference_repo") - if arguments.precision != "bf16": - raise ValueError("upstream-lance requires bf16 precision") - with tempfile.TemporaryDirectory(prefix="trtmc-perf-lance-adapter-") as temporary: - output = Path(temporary) / "result.json" - command = [ - sys.executable, - str(REPOSITORY / "apps/benchmark/performance/baselines/lance_reference.py"), - "--reference-repo", - reference_repo, - "--model", - arguments.model, - "--model-subdir", - str(options.get("model_subdir", "Lance_3B")), - "--vit-subdir", - str(options.get("vit_subdir", "Qwen2.5-VL-ViT")), - "--image", - str(_asset_path(arguments, request, "image_path")), - "--prompt", - str(request.get("prompt", "")), - "--instruction", - str( - options.get( - "instruction", - "Look at the image carefully and answer the question.", - ) - ), - "--max-new-tokens", - str(request.get("max_new_tokens", 16)), - "--warmup", - str(arguments.warmup), - "--iterations", - str(arguments.iterations), - "--resolution", - str(options.get("resolution", "image_768res")), - "--height", - str(options.get("height", 768)), - "--width", - str(options.get("width", 768)), - "--output", - str(output), - ] - if arguments.revision: - command.extend(["--revision", arguments.revision]) - if arguments.local_files_only: - command.append("--local-files-only") - completed = subprocess.run( - command, - cwd=reference_repo, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - f"Lance reference failed with rc={completed.returncode}: {completed.stderr[-2000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - samples = [float(value) for value in payload.get("samples_ms", [])] - if len(samples) != arguments.iterations: - raise RuntimeError( - f"Lance reference returned {len(samples)} samples; expected {arguments.iterations}" - ) - return ( - samples, - {"text": str(payload.get("text", "")), "output_tokens": None}, - "lance-pytorch", - "task-pipeline-call-wall", - True, - True, - ) - - -def _run_lerobot_act( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - source_root = str(options.get("source_root", "")) - if not source_root: - raise ValueError("LeRobot ACT reference requires adapter_options.source_root") - source_root = _reference_checkout( - source_root, repository="https://github.com/huggingface/lerobot" - ) - if arguments.operation != "control" or arguments.precision != "fp32": - raise ValueError("LeRobot ACT performance requires control in fp32") - - with tempfile.TemporaryDirectory(prefix="trtmc-perf-lerobot-act-") as temporary: - output = Path(temporary) / "result.json" - command = [ - sys.executable, - str(REPOSITORY / "families/lerobot_act/tests/performance_reference.py"), - "--source-root", - source_root, - "--model", - arguments.model, - "--image", - str(_asset_path(arguments, request, "image_path")), - "--state", - str(_asset_path(arguments, request, "state_path")), - "--warmup", - str(arguments.warmup), - "--iterations", - str(arguments.iterations), - "--output", - str(output), - ] - if arguments.revision: - command.extend(("--revision", arguments.revision)) - if arguments.local_files_only: - command.append("--local-files-only") - completed = subprocess.run( - command, - cwd=REPOSITORY, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - "LeRobot ACT reference failed with " - f"rc={completed.returncode}: {completed.stderr[-2000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - samples = [float(value) for value in payload.get("samples_ms", [])] - if len(samples) != arguments.iterations: - raise RuntimeError( - f"LeRobot ACT reference returned {len(samples)} samples; " - f"expected {arguments.iterations}" - ) - summary = payload.get("output_summary") - if not isinstance(summary, Mapping): - raise RuntimeError("LeRobot ACT reference returned no output summary") - return ( - samples, - dict(summary), - str(payload.get("framework", "lerobot-pytorch")), - "task-pipeline-call-wall", - True, - False, - ) - - -def _run_sana_wm( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - options = dict(options) - testcase_name = getattr(arguments, "testcase_name", None) - if testcase_name is not None: - manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) - matches = [ - value - for value in manifest.get("testcases", []) - if isinstance(value, Mapping) and value.get("name") == testcase_name - ] - if len(matches) != 1: - raise ValueError(f"SANA-WM requires exactly one testcase named {testcase_name!r}") - for name in ( - "translation_speed", - "rotation_speed_deg", - "fps", - "flow_shift", - "no_action_overlay", - ): - if name in matches[0]: - options.setdefault(name, matches[0][name]) - reference_repo = str(options.get("reference_repo", "")) - if not reference_repo: - raise ValueError("upstream-sana-wm requires baseline.adapter_options.reference_repo") - reference_repo = _reference_checkout( - reference_repo, repository="https://github.com/NVlabs/Sana" - ) - - model_root = arguments.manifest.resolve().parent.parent - - def model_input(value: object) -> Path: - path = Path(str(value)) - return path if path.is_absolute() else (model_root / path).resolve() - - image = model_input(request["image_path"]) - prompt_text = str(request.get("prompt", "")) - if not prompt_text: - raise ValueError("SANA-WM request requires prompt") - action = request.get("action") - if not isinstance(action, str) or not action: - raise ValueError("SANA-WM request requires a non-empty action") - intrinsics = model_input( - options.get("intrinsics_path", options.get("intrinsics", "assets/demo_0_intrinsics.npy")) - ) - for label, path in (("image", image), ("intrinsics", intrinsics)): - if not path.is_file(): - raise FileNotFoundError(f"SANA-WM {label} input does not exist: {path}") - - num_frames = ( - request["num_frames"] - if "num_frames" in request - else json.loads(arguments.manifest.read_text(encoding="utf-8"))["video_num_frames"] - ) - with tempfile.TemporaryDirectory(prefix="trtmc-perf-sana-wm-") as temporary: - root = Path(temporary) - output = root / "benchmark.json" - prompt = root / "prompt.txt" - prompt.write_text(prompt_text, encoding="utf-8") - command = [ - sys.executable, - str(REPOSITORY / "apps/benchmark/performance/baselines/sana_wm_reference.py"), - "--reference-repo", - reference_repo, - "--image", - str(image), - "--model-dir", - str(options["model_dir"]), - "--prompt", - str(prompt), - "--action", - action, - "--intrinsics", - str(intrinsics), - "--translation_speed", - str( - request["translation_speed"] - if "translation_speed" in request - else options["translation_speed"] - ), - "--rotation_speed_deg", - str( - request["rotation_speed_deg"] - if "rotation_speed_deg" in request - else options["rotation_speed_deg"] - ), - "--num_frames", - str(num_frames), - "--fps", - str(request["fps"] if "fps" in request else options["fps"]), - "--step", - str(request["num_steps"]), - "--cfg_scale", - str(request["cfg_scale"]), - "--flow_shift", - str(request["flow_shift"] if "flow_shift" in request else options["flow_shift"]), - "--seed", - str(request["seed"]), - "--refiner_seed", - str(request["seed"]), - "--warmup", - str(arguments.warmup), - "--iterations", - str(arguments.iterations), - "--output", - str(output), - ] - if ( - request["no_action_overlay"] - if "no_action_overlay" in request - else options["no_action_overlay"] - ): - command.append("--no_action_overlay") - completed = subprocess.run( - command, - cwd=reference_repo, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - "SANA-WM reference failed with " - f"rc={completed.returncode}: {completed.stderr[-4000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - summary = dict(payload.get("output_summary", {})) - source_artifacts = summary.get("frame_artifacts", []) - if source_artifacts: - destination = arguments.output.with_suffix(".media").resolve() - destination.mkdir(parents=True, exist_ok=True) - copied = [] - for source_value in source_artifacts: - source = Path(str(source_value)) - target = destination / source.name - shutil.copyfile(source, target) - copied.append(str(target)) - summary["frame_artifacts"] = copied - payload["output_summary"] = summary - samples = [float(value) for value in payload.get("samples_ms", [])] - if len(samples) != arguments.iterations: - raise RuntimeError( - f"SANA-WM reference returned {len(samples)} samples; expected {arguments.iterations}" - ) - summary = dict(payload.get("output_summary", {})) - return ( - samples, - summary, - "sana-wm-pytorch", - "task-pipeline-call-wall", - True, - False, - ) - - -def _environment() -> dict[str, Any]: - value: dict[str, Any] = { - "platform": platform.platform(), - "python": platform.python_version(), - "python_executable": sys.executable, - } - try: - import torch - - value["torch"] = torch.__version__ - if torch.cuda.is_available(): - value["gpu"] = torch.cuda.get_device_name(0) - value["cuda"] = torch.version.cuda - except ImportError: - pass - for module_name in ("transformers", "diffusers", "nemo", "chronos", "moshi"): - try: - module = __import__(module_name) - except ImportError: - continue - value[module_name] = str(getattr(module, "__version__", "unknown")) - return value - - -def run(arguments: argparse.Namespace) -> int: - if arguments.warmup < 0 or arguments.iterations <= 0: - raise ValueError("warmup must be non-negative and iterations must be positive") - options = _json_object(arguments.adapter_options_json, "--adapter-options-json") - expected_mode = "pytorch-eager" if arguments.adapter in PYTORCH_ADAPTERS else "hf-eager" - supported_modes = {expected_mode} - if ( - arguments.adapter == "pytorch-timeseries" - and options.get("reference_type") == "chronos-bolt" - ): - supported_modes.add("torch-compile") - if arguments.adapter == "hf-transformers-embedding": - supported_modes.add("torch-compile") - if arguments.mode not in supported_modes: - raise ValueError(f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}") - request = flatten_config(_json_object(arguments.request_json, "--request-json")) - configured_timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") - fields = ("timing_scope", "input_preparation_included", "asset_loading_included") - expected_timing = None - if configured_timing: - if set(configured_timing) != set(fields): - raise ValueError( - "--timing-contract-json must declare exactly the three reference timing fields" - ) - declared = timing_contract(runner="task-reference", declared=configured_timing) - expected_timing = {name: declared[name] for name in fields} - load_started = time.perf_counter() - load_seconds: float | None = None - compile_evidence: dict[str, Any] | None = None - if arguments.adapter == "upstream-elf": - samples, output_summary, framework, timing_scope, input_included, asset_included = _run_elf( - arguments, request, options - ) - elif arguments.adapter == "upstream-lance": - ( - samples, - output_summary, - framework, - timing_scope, - input_included, - asset_included, - ) = _run_lance(arguments, request, options) - elif arguments.adapter == "pytorch-lerobot-act": - ( - samples, - output_summary, - framework, - timing_scope, - input_included, - asset_included, - ) = _run_lerobot_act(arguments, request, options) - elif arguments.adapter == "upstream-sana-wm": - ( - samples, - output_summary, - framework, - timing_scope, - input_included, - asset_included, - ) = _run_sana_wm(arguments, request, options) - else: - session = LOADERS[arguments.adapter](arguments, request, options) - compile_evidence = session.compile_evidence - load_seconds = time.perf_counter() - load_started - framework = session.framework - timing_scope = session.timing_scope - input_included = session.input_preparation_included - asset_included = session.asset_loading_included - actual_timing = { - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - } - timing_contract(runner="task-reference", declared=actual_timing) - if expected_timing is not None and actual_timing != expected_timing: - raise RuntimeError( - f"{arguments.family} reference implementation timing drifted: " - f"actual={actual_timing}, declared={expected_timing}" - ) - samples, output_summary = _measure(session, arguments.warmup, arguments.iterations) - if "_media" in output_summary: - _write_media_artifacts(output_summary, arguments.output) - if "_geometry_arrays" in output_summary: - _write_geometry_artifacts(output_summary, arguments.output) - disparity = output_summary.pop("_disparity_f32", None) - if disparity is not None: - artifact_path = arguments.output.with_suffix(".disparity.f32").resolve() - artifact_path.parent.mkdir(parents=True, exist_ok=True) - disparity.tofile(artifact_path) - output_summary["disparity_artifact"] = str(artifact_path) - audio = output_summary.pop("_audio_f32", None) - if audio is not None: - import soundfile as sf - - artifact_path = arguments.output.with_suffix(".audio.wav").resolve() - artifact_path.parent.mkdir(parents=True, exist_ok=True) - sf.write(artifact_path, audio, output_summary["sample_rate"], subtype="FLOAT") - output_summary["audio_artifact"] = str(artifact_path) - if arguments.adapter in { - "pytorch-lerobot-act", - "upstream-elf", - "upstream-lance", - "upstream-sana-wm", - }: - actual_timing = { - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - } - timing_contract(runner="task-reference", declared=actual_timing) - if expected_timing is not None and actual_timing != expected_timing: - raise RuntimeError( - f"{arguments.family} reference implementation timing drifted: " - f"actual={actual_timing}, declared={expected_timing}" - ) - result = { - "schema_version": "trtmc.perf-baseline/v1", - "status": "completed", - "backend": arguments.adapter, - "adapter": arguments.adapter, - "framework": framework, - "mode": arguments.mode, - "precision": arguments.precision, - "padding": arguments.padding, - "experts_implementation": None, - "compile_scope": "model.forward" if arguments.mode == "torch-compile" else None, - "compile_evidence": compile_evidence, - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - "model_load_included": False, - "model_load_seconds": load_seconds, - "model": arguments.model, - "case_name": arguments.case_name, - "measurement": { - "warmup": arguments.warmup, - "iterations": arguments.iterations, - }, - "measurement_policy": { - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - "model_load_excluded": True, - "compile_excluded": True, - "warmup_excluded": True, - "output_materialization_included": True, - }, - "samples_ms": samples, - "metrics": { - "sample_count": len(samples), - "latency_ms": { - "p50": statistics.median(samples), - "min": min(samples), - "max": max(samples), - "mean": statistics.fmean(samples), - }, - }, - "output_summary": output_summary, - "environment": _environment(), - "finished_at": datetime.now(timezone.utc).isoformat(), - } - if compile_evidence is not None: - compile_evidence["warmup_completed"] = True - compile_evidence["timed_callable_uses_compiled_target"] = True - if not all(math.isfinite(float(value)) and float(value) > 0.0 for value in samples): - raise RuntimeError("reference produced an invalid timing sample") - arguments.output.parent.mkdir(parents=True, exist_ok=True) - arguments.output.write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - return 0 - - -def main(argv: Sequence[str] | None = None) -> int: - try: - arguments = build_parser().parse_args(argv) - if arguments.local_files_only: - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ["TRANSFORMERS_OFFLINE"] = "1" - return run(arguments) - except (KeyError, OSError, RuntimeError, TypeError, ValueError) as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py b/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py index 964618daa6..f91fcab68b 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - """Behavioral controls for report evidence and managed bundle reuse.""" from __future__ import annotations @@ -761,50 +760,3 @@ def test_report_image_artifacts_obey_case_boundary_and_do_not_invent_time(tmp_pa assert "outside.png" not in document and "link.png" not in document assert "Output images: 4 artifact(s) unavailable" in document assert "Frame 0" in document and "Frame 0 ·" not in document - - -@pytest.mark.parametrize( - "schema, row_key, reference_key", [("v1", "cases", "baseline"), ("v2", "rows", "reference")] -) -def test_performance_report_preserves_historical_measurements( - tmp_path, schema, row_key, reference_key -): - from tools import perf_matrix - - scope = "model_call_wall" if schema == "v1" else "public_task_call_wall" - results = { - "schema_version": "trtmc.perf-matrix/" + schema, - "status": "completed", - "selected_entry_ids": ["example.generate"], - row_key: [ - { - "id": "example.generate", - "model": "example", - "operation": "generate", - "status": "green", - "candidate": {"samples_ms": [8.0, 10.0, 12.0], "timing_scope": scope}, - reference_key: { - "samples_ms": [12.0, 14.0, 16.0], - "measurement_policy": {"timing_scope": "public_operation_call_wall"}, - }, - "commands": {"candidate": {"argv": ["trtmc-bench", "run", "--model", "example"]}}, - } - ], - } - original = json.dumps(results) - (tmp_path / "results.json").write_text(original) - assert perf_matrix.main(["report", str(tmp_path)]) == 0 - report = json.loads((tmp_path / "report.json").read_text()) - document = (tmp_path / "report.html").read_text() - assert report["source_schema_version"] == results["schema_version"] - assert report["rows"][0][reference_key] == results[row_key][0][reference_key] - assert "p50: 10.000 ms" in document - assert "p95: 11.800 ms" in document - assert scope in document - assert "public_operation_call_wall" in document - assert "trtmc-bench" in document - assert 'id="filter"' in document - assert (tmp_path / "results.json").read_text() == original - if schema == "v1": - with pytest.raises(perf_matrix.PerfMatrixError, match="unsupported results schema"): - perf_matrix._load_results(tmp_path) diff --git a/families/canary/tests/benchmark/canary-1b-v2.yaml b/families/canary/tests/benchmark/canary-1b-v2.yaml index f526943067..07bcd38999 100644 --- a/families/canary/tests/benchmark/canary-1b-v2.yaml +++ b/families/canary/tests/benchmark/canary-1b-v2.yaml @@ -42,7 +42,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: nemo-asr + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference precision: fp32 diff --git a/families/canary/tests/benchmark/reference.py b/families/canary/tests/benchmark/reference.py index 2c183bb216..96b04f3b14 100644 --- a/families/canary/tests/benchmark/reference.py +++ b/families/canary/tests/benchmark/reference.py @@ -7,16 +7,26 @@ from __future__ import annotations from pathlib import Path -from typing import Any, Sequence +import tempfile +import atexit +import sys +from typing import Any, Mapping, Sequence -from tools.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_audio_mono, + resample_linear, + transcription_text, + write_wav_pcm16, +) def _text(value: Any) -> str: return str(value.text if hasattr(value, "text") else value) -def main(argv: Sequence[str] | None = None) -> int: +def _accuracy(argv: Sequence[str] | None = None) -> int: arguments = speech_io.parser().parse_args(argv) request = speech_io.load_request(arguments.request) samples = speech_io.prepare_samples(request, arguments.output) @@ -33,7 +43,7 @@ def main(argv: Sequence[str] | None = None) -> int: archives = sorted(Path(snapshot).glob("*.nemo")) if not archives: raise FileNotFoundError(f"Canary NeMo archive is missing for {model_id}") - model = ASRModel.restore_from(str(archives[0]), map_location="cpu").cpu().eval() + model = ASRModel.restore_from(str(archives[0]), map_location="cpu").eval().to("cuda") values = model.transcribe([str(sample.wav_path) for sample in samples], batch_size=1) if isinstance(values, tuple): values = values[0] @@ -41,5 +51,53 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 +def _performance_session( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager" or arguments.precision != "fp32": + raise ValueError("Canary reference requires pytorch-eager fp32") + from huggingface_hub import snapshot_download + from nemo.collections.asr.models import ASRModel + + snapshot = snapshot_download( + repo_id=arguments.model, + revision=arguments.revision, + allow_patterns=["*.nemo"], + local_files_only=arguments.local_files_only, + ) + archives = sorted(Path(snapshot).glob("*.nemo")) + if not archives: + raise FileNotFoundError(f"Canary NeMo archive is missing for {arguments.model}") + model = ASRModel.restore_from(str(archives[0]), map_location="cpu").cpu().eval() + audio, source_rate = load_audio_mono(str(request["audio_path"])) + target_rate = 16_000 + audio = resample_linear(audio, source_rate, target_rate) + temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + temporary.close() + atexit.register(Path(temporary.name).unlink, missing_ok=True) + write_wav_pcm16(Path(temporary.name), audio, target_rate) + + def invoke() -> Mapping[str, Any]: + values = model.transcribe([temporary.name], batch_size=1) + return {"text": transcription_text(values), "output_tokens": None} + + return reference_harness.Session( + invoke, + "nemo", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml index b8f21ef425..b79012a2ae 100644 --- a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml +++ b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml @@ -48,8 +48,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-transformers-vlm - adapter_options: {workflow: document-ocr-infer} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: bf16 diff --git a/families/deepseek_ocr/tests/benchmark/reference.py b/families/deepseek_ocr/tests/benchmark/reference.py index a84192ae1f..eb4076c6a5 100644 --- a/families/deepseek_ocr/tests/benchmark/reference.py +++ b/families/deepseek_ocr/tests/benchmark/reference.py @@ -7,10 +7,16 @@ from __future__ import annotations import argparse +import atexit import json import os from pathlib import Path +import shutil +import sys import tempfile +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness def _model_directory(model: str, revision: str | None) -> Path: @@ -28,11 +34,11 @@ def _model_directory(model: str, revision: str | None) -> Path: ) -def main() -> int: +def _run_accuracy(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--request", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) - arguments = parser.parse_args() + arguments = parser.parse_args(argv) request = json.loads(arguments.request.read_text(encoding="utf-8")) import torch @@ -92,5 +98,74 @@ def main() -> int: return 0 +def _load_performance( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from transformers import AutoModel, AutoTokenizer + + if arguments.precision != "bf16": + raise ValueError("DeepSeek OCR official reference requires bf16") + model_dir = _model_directory(arguments.model, arguments.revision) + tokenizer = AutoTokenizer.from_pretrained( + model_dir, + trust_remote_code=arguments.trust_remote_code, + local_files_only=arguments.local_files_only, + ) + model = ( + AutoModel.from_pretrained( + model_dir, + trust_remote_code=arguments.trust_remote_code, + local_files_only=arguments.local_files_only, + use_safetensors=True, + torch_dtype=torch.bfloat16, + attn_implementation="eager", + ) + .to("cuda") + .eval() + ) + image_path = str(request["image_path"]) + prompt = str(request.get("prompt", "")) + if "" not in prompt: + prompt = f"\n{prompt}" + output_directory = Path(tempfile.mkdtemp(prefix="deepseek-ocr-performance-")) + atexit.register(shutil.rmtree, output_directory, ignore_errors=True) + + def invoke() -> Mapping[str, Any]: + text = str( + model.infer( + tokenizer, + prompt=prompt, + image_file=image_path, + output_path=str(output_directory), + base_size=1024, + image_size=768, + crop_mode=True, + save_results=False, + eval_mode=True, + ) + or "" + ) + token_ids = tokenizer(text, add_special_tokens=False).input_ids + return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} + + return reference_harness.Session( + invoke, + "transformers", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if "--request" in arguments: + return _run_accuracy(arguments) + return reference_harness.run(arguments, description=__doc__, load=_load_performance) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/deepseek_ocr/tests/test_benchmark_reference.py b/families/deepseek_ocr/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..6823e9e8c8 --- /dev/null +++ b/families/deepseek_ocr/tests/test_benchmark_reference.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/deepseek-ocr.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert reference["output_contract"] == "ocr-text" + assert "adapter" not in reference diff --git a/families/detr/tests/benchmark/detr-resnet-50.yaml b/families/detr/tests/benchmark/detr-resnet-50.yaml index 6b2c77093c..805afa31cf 100644 --- a/families/detr/tests/benchmark/detr-resnet-50.yaml +++ b/families/detr/tests/benchmark/detr-resnet-50.yaml @@ -44,8 +44,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: object-detection} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/detr/tests/benchmark/reference.py b/families/detr/tests/benchmark/reference.py index aca5be52de..75e3a71612 100644 --- a/families/detr/tests/benchmark/reference.py +++ b/families/detr/tests/benchmark/reference.py @@ -9,7 +9,10 @@ import argparse import json from pathlib import Path -from typing import Any, Sequence +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness def _parser() -> argparse.ArgumentParser: @@ -26,7 +29,7 @@ def _dtype(torch: Any, precision: str) -> Any: raise ValueError(f"unsupported reference precision {precision!r}") from error -def main(argv: Sequence[str] | None = None) -> int: +def _accuracy(argv: Sequence[str] | None = None) -> int: arguments = _parser().parse_args(argv) request = json.loads(arguments.request.read_text(encoding="utf-8")) samples = request.get("samples") @@ -84,5 +87,70 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 +def _performance_session( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForObjectDetection + + if arguments.mode != "hf-eager": + raise ValueError("DETR reference requires hf-eager mode") + image = Image.open(str(request["image_path"])).convert("RGB") + options = {"revision": arguments.revision} if arguments.revision else {} + processor = AutoImageProcessor.from_pretrained(arguments.model, **options) + model = ( + AutoModelForObjectDetection.from_pretrained( + arguments.model, + torch_dtype=_dtype(torch, arguments.precision), + **options, + ) + .eval() + .to("cuda") + ) + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + threshold = float(request.get("score_threshold", 0.5)) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + detections = processor.post_process_object_detection( + outputs, + threshold=threshold, + target_sizes=torch.tensor([[image.height, image.width]], device=model.device), + )[0] + scores = detections["scores"].float().cpu() + return { + "detected_images": 1, + "detections": int(scores.shape[0]), + "image_height": image.height, + "image_width": image.width, + "boxes": detections["boxes"].float().cpu().reshape(-1).tolist(), + "scores": scores.tolist(), + "class_ids": detections["labels"].cpu().tolist(), + "shape": [int(scores.shape[0]), 4], + "coordinates": "xyxy", + "units": "pixels", + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml index aaee48a41f..b60ff4c0ea 100644 --- a/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml +++ b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml @@ -41,8 +41,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: image-features} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml index 58a866c8d7..cc602ac830 100644 --- a/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml +++ b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml @@ -41,8 +41,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: image-features} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/dinov3/tests/benchmark/reference.py b/families/dinov3/tests/benchmark/reference.py index 469e2ee307..d414a8b0d9 100644 --- a/families/dinov3/tests/benchmark/reference.py +++ b/families/dinov3/tests/benchmark/reference.py @@ -9,7 +9,10 @@ import argparse import json from pathlib import Path -from typing import Any, Sequence +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness def _parser() -> argparse.ArgumentParser: @@ -26,7 +29,7 @@ def _dtype(torch: Any, precision: str) -> Any: raise ValueError(f"unsupported reference precision {precision!r}") from error -def main(argv: Sequence[str] | None = None) -> int: +def _accuracy(argv: Sequence[str] | None = None) -> int: arguments = _parser().parse_args(argv) request = json.loads(arguments.request.read_text(encoding="utf-8")) samples = request.get("samples") @@ -74,5 +77,55 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 +def _performance_session( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModel + + if arguments.mode != "hf-eager": + raise ValueError("DINOv3 reference requires hf-eager mode") + options = {"revision": arguments.revision} if arguments.revision else {} + processor = AutoImageProcessor.from_pretrained(arguments.model, **options) + model = ( + AutoModel.from_pretrained( + arguments.model, + torch_dtype=_dtype(torch, arguments.precision), + **options, + ) + .eval() + .to("cuda") + ) + image = Image.open(str(request["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + return { + "last_hidden_state_shape": [int(size) for size in outputs.last_hidden_state.shape], + "pooler_output_shape": [int(size) for size in outputs.pooler_output.shape], + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml b/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml index d840cf93c3..168766e944 100644 --- a/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml +++ b/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml @@ -41,7 +41,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-chat-asr + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/glmasr/tests/benchmark/reference.py b/families/glmasr/tests/benchmark/reference.py index d79c83474f..2d960224bb 100644 --- a/families/glmasr/tests/benchmark/reference.py +++ b/families/glmasr/tests/benchmark/reference.py @@ -6,12 +6,18 @@ from __future__ import annotations -from typing import Sequence +import sys +from typing import Any, Mapping, Sequence -from tools.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_audio_mono, + resample_linear, +) -def main(argv: Sequence[str] | None = None) -> int: +def _accuracy(argv: Sequence[str] | None = None) -> int: arguments = speech_io.parser().parse_args(argv) request = speech_io.load_request(arguments.request) samples = speech_io.prepare_samples(request, arguments.output) @@ -22,11 +28,15 @@ def main(argv: Sequence[str] | None = None) -> int: model_id, revision = speech_io.model_id(request) options = {"revision": revision} if revision else {} processor = GlmAsrProcessor.from_pretrained(model_id, **options) - model = GlmAsrForConditionalGeneration.from_pretrained( - model_id, - torch_dtype=speech_io.torch_dtype(torch, request), - **options, - ).eval().to("cuda") + model = ( + GlmAsrForConditionalGeneration.from_pretrained( + model_id, + torch_dtype=speech_io.torch_dtype(torch, request), + **options, + ) + .eval() + .to("cuda") + ) max_new_tokens = int(request.get("max_new_tokens", 128)) texts = [] for sample in samples: @@ -55,5 +65,74 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 +def _performance_session( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "hf-eager": + raise ValueError("GLM-ASR reference requires hf-eager mode") + import torch + from transformers import GlmAsrForConditionalGeneration, GlmAsrProcessor + + options = {"revision": arguments.revision} if arguments.revision else {} + processor = GlmAsrProcessor.from_pretrained(arguments.model, **options) + dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[ + arguments.precision + ] + model = ( + GlmAsrForConditionalGeneration.from_pretrained( + arguments.model, torch_dtype=dtype, **options + ) + .eval() + .to("cuda") + ) + audio, source_rate = load_audio_mono(str(request["audio_path"])) + target_rate = 16_000 + audio = resample_linear(audio, source_rate, target_rate) + conversation = [ + { + "role": "user", + "content": [ + {"type": "audio", "audio": audio}, + {"type": "text", "text": "Please transcribe this audio into text"}, + ], + } + ] + inputs = processor.apply_chat_template( + conversation, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, + sampling_rate=target_rate, + ).to(model.device) + prompt_tokens = int(inputs["input_ids"].shape[1]) + max_new_tokens = int(request.get("max_new_tokens", 128)) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + generated = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + ) + token_ids = [int(token) for token in generated[0, prompt_tokens:].cpu().tolist()] + return { + "text": processor.tokenizer.decode(token_ids, skip_special_tokens=True).strip(), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml index bcb562c623..c43d4da9e9 100644 --- a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml +++ b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml @@ -12,7 +12,7 @@ accuracy: benchmark: task_output_parity operation: generate request: &request {prompt: "What color is the vehicle in this image? Answer in one word.", image_path: ../data/test_img.jpeg, max_new_tokens: 10} - reference: &reference {adapter: upstream-lance, mode: pytorch-eager, precision: bf16, output_contract: normalized-text, adapter_options: {model_subdir: Lance_3B, vit_subdir: Qwen2.5-VL-ViT, resolution: image_768res}} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: bf16, output_contract: normalized-text, adapter_options: {model_subdir: Lance_3B, vit_subdir: Qwen2.5-VL-ViT, resolution: image_768res}} gate: {max_normalized_edit_distance: 0.15} performance: - {name: generate, benchmark: vision_language_pipeline_performance, operation: generate, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: upstream_lance, output_contract: exact-text}} diff --git a/families/lance/tests/benchmark/reference.py b/families/lance/tests/benchmark/reference.py new file mode 100644 index 0000000000..1dd3d0abf3 --- /dev/null +++ b/families/lance/tests/benchmark/reference.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned Lance reference.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tempfile +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _execute( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Measurement: + reference_repo = Path(str(options.get("reference_repo", ""))).resolve() + if not reference_repo.is_dir(): + raise ValueError("Lance reference requires reference_repo") + if arguments.mode != "pytorch-eager" or arguments.precision != "bf16": + raise ValueError("Lance reference requires pytorch-eager bf16") + with tempfile.TemporaryDirectory(prefix="trtmc-perf-lance-") as temporary: + output = Path(temporary) / "result.json" + command = [ + sys.executable, + str(Path(__file__).with_name("upstream_reference.py")), + "--reference-repo", + str(reference_repo), + "--model", + arguments.model, + "--model-subdir", + str(options.get("model_subdir", "Lance_3B")), + "--vit-subdir", + str(options.get("vit_subdir", "Qwen2.5-VL-ViT")), + "--image", + str(request["image_path"]), + "--prompt", + str(request.get("prompt", "")), + "--instruction", + str(options.get("instruction", "Look at the image carefully and answer the question.")), + "--max-new-tokens", + str(request.get("max_new_tokens", 16)), + "--warmup", + str(arguments.warmup), + "--iterations", + str(arguments.iterations), + "--resolution", + str(options.get("resolution", "image_768res")), + "--height", + str(options.get("height", 768)), + "--width", + str(options.get("width", 768)), + "--output", + str(output), + ] + if arguments.revision: + command.extend(("--revision", arguments.revision)) + if arguments.local_files_only: + command.append("--local-files-only") + completed = subprocess.run( + command, + cwd=reference_repo, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + raise RuntimeError( + f"Lance reference failed with rc={completed.returncode}: {completed.stderr[-2000:]}" + ) + payload = json.loads(output.read_text(encoding="utf-8")) + return reference_harness.Measurement( + [float(value) for value in payload.get("samples_ms", [])], + {"text": str(payload.get("text", "")), "output_tokens": None}, + "lance-pytorch", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run_premeasured( + list(sys.argv[1:] if argv is None else argv), + description=__doc__, + execute=_execute, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/benchmark/performance/baselines/lance_reference.py b/families/lance/tests/benchmark/upstream_reference.py similarity index 100% rename from apps/benchmark/performance/baselines/lance_reference.py rename to families/lance/tests/benchmark/upstream_reference.py diff --git a/families/lance/tests/test_benchmark_reference.py b/families/lance/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..49579f82e2 --- /dev/null +++ b/families/lance/tests/test_benchmark_reference.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + +from families.lance.tests.benchmark import upstream_reference + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_uses_the_family_owned_reference(tmp_path: Path) -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/lance-3b-x2t-image.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert "adapter" not in reference + + checkout = tmp_path / "Lance" + parsed = upstream_reference.build_parser().parse_args( + [ + "--reference-repo", + str(checkout), + "--model", + "model", + "--image", + str(tmp_path / "image.png"), + "--prompt", + "describe", + "--max-new-tokens", + "4", + "--warmup", + "1", + "--iterations", + "2", + "--output", + str(tmp_path / "output.json"), + ] + ) + assert parsed.reference_repo == checkout diff --git a/families/lerobot_act/tests/test_benchmark_reference.py b/families/lerobot_act/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..8d92ca6849 --- /dev/null +++ b/families/lerobot_act/tests/test_benchmark_reference.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/act-aloha-sim-transfer-cube.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert reference["output_contract"] == "robot-action-parity" + assert "adapter" not in reference diff --git a/families/locateanything/tests/benchmark/locateanything-3b.yaml b/families/locateanything/tests/benchmark/locateanything-3b.yaml index 0a7ef06b87..88a361eacb 100644 --- a/families/locateanything/tests/benchmark/locateanything-3b.yaml +++ b/families/locateanything/tests/benchmark/locateanything-3b.yaml @@ -50,8 +50,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-transformers-vlm - adapter_options: {workflow: grounded-generation} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/locateanything/tests/benchmark/reference.py b/families/locateanything/tests/benchmark/reference.py index 5b4b8669c1..d4c9a9d06a 100644 --- a/families/locateanything/tests/benchmark/reference.py +++ b/families/locateanything/tests/benchmark/reference.py @@ -10,6 +10,10 @@ import json import os from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness from families.locateanything.tests.hf_reference import ( _LocalTokenizer, @@ -77,11 +81,11 @@ def _infer(model, tokenizer, image_path: str, prompt: str, max_new_tokens: int) return str(text).strip() -def main() -> int: +def _run_accuracy(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--request", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) - arguments = parser.parse_args() + arguments = parser.parse_args(argv) request = json.loads(arguments.request.read_text(encoding="utf-8")) import torch @@ -128,5 +132,51 @@ def main() -> int: return 0 +def _load_performance( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from transformers import AutoModel + + if arguments.precision != "fp32": + raise ValueError("LocateAnything official reference requires fp32") + model_dir = _model_directory(arguments.model, arguments.revision) + config = _load_config(model_dir) + tokenizer = _LocalTokenizer(model_dir) + model = ( + AutoModel.from_pretrained( + model_dir, + config=config, + trust_remote_code=arguments.trust_remote_code, + local_files_only=arguments.local_files_only, + torch_dtype=torch.float32, + ) + .to("cuda") + .eval() + ) + _repair_rotary_buffers(model) + image_path = str(request["image_path"]) + prompt = str(request.get("prompt", "")) + max_new_tokens = int(request.get("max_new_tokens", 32)) + + def invoke() -> Mapping[str, Any]: + text = _infer(model, tokenizer, image_path, prompt, max_new_tokens) + if not text: + raise RuntimeError("LocateAnything reference produced empty text") + token_ids = tokenizer.encode(text, add_special_tokens=False) + return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if "--request" in arguments: + return _run_accuracy(arguments) + return reference_harness.run(arguments, description=__doc__, load=_load_performance) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/locateanything/tests/test_benchmark_reference.py b/families/locateanything/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..0597913311 --- /dev/null +++ b/families/locateanything/tests/test_benchmark_reference.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/locateanything-3b.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert reference["output_contract"] == "localization" + assert "adapter" not in reference diff --git a/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml b/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml index 92265c9293..1b657ddf26 100644 --- a/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml +++ b/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml @@ -9,7 +9,7 @@ accuracy: benchmark: task_output_parity operation: generate_audio request: &request {prompt: Please say this sentence clearly., max_new_tokens: 260, seed: 42} - reference: &reference {adapter: nemo-tts, mode: pytorch-eager, precision: fp32, output_contract: audio-artifact-parity, adapter_options: {speaker_encoder_revision: e9124b5364a2c3e9b4f78da429a33cbca8f8c22b}} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: fp32, output_contract: audio-artifact-parity, adapter_options: {speaker_encoder_revision: e9124b5364a2c3e9b4f78da429a33cbca8f8c22b}} gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} performance: - {name: generate-audio, benchmark: audio_generation_pipeline_performance, operation: generate_audio, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: nemo_reference, output_contract: audio-shape}} diff --git a/families/magpie_tts/tests/benchmark/reference.py b/families/magpie_tts/tests/benchmark/reference.py new file mode 100644 index 0000000000..f9bc1efb41 --- /dev/null +++ b/families/magpie_tts/tests/benchmark/reference.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned Magpie TTS reference.""" + +from __future__ import annotations + +import os +from pathlib import Path +import random +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +SPEAKER_REPOSITORY = "Edresson/Speaker_Encoder_H_ASP" +SPEAKER_FILENAME = "pytorch_model.bin" +SPEAKER_URL = "https://huggingface.co/Edresson/Speaker_Encoder_H_ASP/resolve/main/pytorch_model.bin" + + +def _seed(torch: Any, value: int) -> None: + import numpy as np + + random.seed(value) + np.random.seed(value) + torch.manual_seed(value) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(value) + + +def _materialize(summary: dict[str, Any], output: Path) -> None: + import soundfile as sound + + audio = summary.pop("_audio_f32") + artifact = output.with_suffix(".audio.wav").resolve() + artifact.parent.mkdir(parents=True, exist_ok=True) + sound.write(artifact, audio, int(summary["sample_rate"]), subtype="FLOAT") + summary["audio_artifact"] = str(artifact) + + +def _load( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager" or arguments.precision != "fp32": + raise ValueError("Magpie TTS reference requires pytorch-eager fp32") + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + import fsspec + import torch + from huggingface_hub import hf_hub_download + from nemo.collections.tts.models import MagpieTTSModel + + torch.use_deterministic_algorithms(True) + speaker_revision = str(options.get("speaker_encoder_revision", "") or "") + if not speaker_revision: + raise ValueError("Magpie TTS requires speaker_encoder_revision") + speaker_checkpoint = hf_hub_download( + repo_id=SPEAKER_REPOSITORY, + filename=SPEAKER_FILENAME, + revision=speaker_revision, + local_files_only=arguments.local_files_only, + ) + original_open = fsspec.open + + def offline_open(path: Any, *args: Any, **kwargs: Any) -> Any: + if str(path).split("?", 1)[0] == SPEAKER_URL: + path = speaker_checkpoint + return original_open(path, *args, **kwargs) + + fsspec.open = offline_open + archive = hf_hub_download( + repo_id=arguments.model, + filename="magpie_tts_multilingual_357m.nemo", + revision=arguments.revision, + local_files_only=arguments.local_files_only, + ) + model = MagpieTTSModel.restore_from(restore_path=archive).eval().to("cuda") + max_new_tokens = request.get("max_new_tokens", 0) + if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): + raise ValueError("Magpie max_new_tokens must be an integer") + if max_new_tokens > 0: + model.inference_parameters.max_decoder_steps = max_new_tokens + seed = request.get("seed", 42) + if isinstance(seed, bool) or not isinstance(seed, int): + raise ValueError("Magpie seed must be an integer") + prompt = str(request.get("prompt", "")) + + def invoke() -> Mapping[str, Any]: + _seed(torch, seed) + with torch.inference_mode(): + audio, length = model.do_tts(transcript=prompt, language="en", use_cfg=True) + count = int(length.item()) if length.numel() else int(audio.numel()) + return { + "audio_samples": count, + "sample_rate": 22_050, + "_audio_f32": audio.detach().float().cpu().reshape(-1)[:count].numpy(), + } + + return reference_harness.Session( + invoke, + "nemo", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + materialize=_materialize, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run( + list(sys.argv[1:] if argv is None else argv), + description=__doc__, + load=_load, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml index 7159fbb9cc..ab37b766de 100644 --- a/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml @@ -46,8 +46,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: nemo-asr - adapter_options: {transcription_input: manifest, model_loader: prompted-nemo-archive} + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference precision: fp32 diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml index b956e3d29b..b7f8da14f8 100644 --- a/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml @@ -43,8 +43,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: nemo-asr - adapter_options: {transcription_input: manifest} + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference precision: fp32 diff --git a/families/nemotron_speech_streaming/tests/benchmark/reference.py b/families/nemotron_speech_streaming/tests/benchmark/reference.py index 56e40841b2..cfa90178ea 100644 --- a/families/nemotron_speech_streaming/tests/benchmark/reference.py +++ b/families/nemotron_speech_streaming/tests/benchmark/reference.py @@ -7,16 +7,27 @@ from __future__ import annotations import json -from typing import Any, Sequence - -from tools.benchmark_qualification.references import speech_io +from pathlib import Path +import tempfile +import atexit +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_audio_mono, + resample_linear, + transcription_text, + write_wav_pcm16, +) def _text(value: Any) -> str: return str(value.text if hasattr(value, "text") else value) -def main(argv: Sequence[str] | None = None) -> int: +def _accuracy(argv: Sequence[str] | None = None) -> int: arguments = speech_io.parser().parse_args(argv) request = speech_io.load_request(arguments.request) samples = speech_io.prepare_samples(request, arguments.output) @@ -27,7 +38,7 @@ def main(argv: Sequence[str] | None = None) -> int: model_id, revision = speech_io.model_id(request) is_nemotron35 = "nemotron-3.5-asr-streaming" in model_id.casefold() if is_nemotron35: - from apps.benchmark.performance.baselines.audio_reference import ( + from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( load_nemotron35_asr_model, ) @@ -81,5 +92,92 @@ def forward_with_extended_prompt(*args: Any, **kwargs: Any) -> Any: return 0 +def _disable_cuda_graphs(model: Any) -> None: + decoding = getattr(getattr(model, "cfg", None), "decoding", None) + if decoding is not None and hasattr(decoding, "use_cuda_graph_decoder"): + decoding.use_cuda_graph_decoder = False + change_strategy = getattr(model, "change_decoding_strategy", None) + if callable(change_strategy) and decoding is not None: + change_strategy(decoding_cfg=decoding) + + +def _performance_session( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager" or arguments.precision != "fp32": + raise ValueError("Nemotron streaming ASR reference requires pytorch-eager fp32") + import torch + from nemo.collections.asr.models import ASRModel + + is_nemotron35 = "nemotron-3.5-asr-streaming" in arguments.model.casefold() + if is_nemotron35: + from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_nemotron35_asr_model, + ) + + model = load_nemotron35_asr_model( + model=arguments.model, + revision=arguments.revision or "", + local_files_only=arguments.local_files_only, + device="cuda", + ) + else: + model = ASRModel.from_pretrained(arguments.model, map_location="cpu").eval().to("cuda") + _disable_cuda_graphs(model) + audio, source_rate = load_audio_mono(str(request["audio_path"])) + target_rate = 16_000 + audio = resample_linear(audio, source_rate, target_rate) + temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + temporary.close() + atexit.register(Path(temporary.name).unlink, missing_ok=True) + write_wav_pcm16(Path(temporary.name), audio, target_rate) + manifest = Path(temporary.name).with_suffix(".jsonl") + atexit.register(manifest.unlink, missing_ok=True) + record: dict[str, Any] = { + "audio_filepath": temporary.name, + "duration": float(len(audio)) / target_rate, + "text": "", + } + language = str(request.get("language", "") or "") + if language and language != "auto": + record["lang"] = language + manifest.write_text(json.dumps(record) + "\n", encoding="utf-8") + if is_nemotron35: + original_forward = model.forward + + def forward_with_extended_prompt(*args: Any, **kwargs: Any) -> Any: + prompt = kwargs.get("prompt") + if prompt is not None and prompt.shape[1] > 0: + kwargs = dict(kwargs) + kwargs["prompt"] = torch.cat((prompt, prompt[:, -1:, :]), dim=1) + return original_forward(*args, **kwargs) + + model.forward = forward_with_extended_prompt + + def invoke() -> Mapping[str, Any]: + options: dict[str, Any] = {"batch_size": 1} + if is_nemotron35: + options["verbose"] = False + values = model.transcribe(str(manifest), **options) + return {"text": transcription_text(values), "output_tokens": None} + + return reference_harness.Session( + invoke, + "nemo", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/personaplex/tests/benchmark/personaplex-7b.yaml b/families/personaplex/tests/benchmark/personaplex-7b.yaml index ddde6f1a00..23b5ced569 100644 --- a/families/personaplex/tests/benchmark/personaplex-7b.yaml +++ b/families/personaplex/tests/benchmark/personaplex-7b.yaml @@ -11,7 +11,7 @@ accuracy: benchmark: task_output_parity operation: speak request: &request {audio_path: ../data/Recording.wav, max_new_tokens: 300} - reference: &reference {adapter: pytorch-personaplex, mode: pytorch-eager, precision: bf16, output_contract: audio-artifact-parity} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: bf16, output_contract: audio-artifact-parity} gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} performance: - {name: speak, benchmark: audio_generation_pipeline_performance, operation: speak, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: torch_reference, output_contract: audio-shape}} diff --git a/families/personaplex/tests/benchmark/reference.py b/families/personaplex/tests/benchmark/reference.py new file mode 100644 index 0000000000..ef0cb1b69d --- /dev/null +++ b/families/personaplex/tests/benchmark/reference.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned PersonaPlex reference.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _materialize(summary: dict[str, Any], output: Path) -> None: + import soundfile as sound + + audio = summary.pop("_audio_f32") + artifact = output.with_suffix(".audio.wav").resolve() + artifact.parent.mkdir(parents=True, exist_ok=True) + sound.write(artifact, audio, int(summary["sample_rate"]), subtype="FLOAT") + summary["audio_artifact"] = str(artifact) + + +def _load( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager": + raise ValueError("PersonaPlex reference requires pytorch-eager mode") + official_repo = Path(str(options.get("official_repo", ""))).resolve() + if not official_repo.is_dir(): + raise ValueError("PersonaPlex reference requires official_repo") + if importlib.util.find_spec("sphn") is None: + raise RuntimeError("PersonaPlex reference requires the official sphn package") + sys.path[:0] = [str(official_repo / "moshi"), str(official_repo)] + import torch + from huggingface_hub import hf_hub_download + from moshi.models import LMGen, loaders + from moshi.models.lm import _iterate_audio, encode_from_sphn, load_audio + from moshi.offline import warmup + + device = "cuda" + common = { + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + common["revision"] = arguments.revision + mimi_weights = hf_hub_download(arguments.model, loaders.MIMI_NAME, **common) + model_weights = hf_hub_download(arguments.model, loaders.MOSHI_NAME, **common) + mimi = loaders.get_mimi(mimi_weights, device) + other_mimi = loaders.get_mimi(mimi_weights, device) + language_model = loaders.get_moshi_lm(model_weights, device=device).eval() + frame_size = int(mimi.sample_rate / mimi.frame_rate) + generator = LMGen( + language_model, + audio_silence_frame_cnt=0, + sample_rate=mimi.sample_rate, + device=device, + frame_rate=mimi.frame_rate, + use_sampling=False, + temp=0.8, + temp_text=0.7, + top_k=250, + top_k_text=25, + ) + mimi.streaming_forever(1) + other_mimi.streaming_forever(1) + generator.streaming_forever(1) + warmup(mimi, other_mimi, generator, device, frame_size) + source_audio = load_audio(str(request["audio_path"]), mimi.sample_rate) + max_frames = int(request.get("max_new_tokens", options.get("max_frames", 100))) + + def invoke() -> Mapping[str, Any]: + mimi.reset_streaming() + other_mimi.reset_streaming() + generator.reset_streaming() + generated_frames = 0 + decoded_chunks = [] + + def summary() -> Mapping[str, Any]: + audio = ( + torch.cat(decoded_chunks, dim=-1).detach().float().cpu().reshape(-1).numpy() + if decoded_chunks + else torch.empty(0, dtype=torch.float32).numpy() + ) + return { + "audio_frames": generated_frames, + "audio_samples": int(audio.size), + "sample_rate": mimi.sample_rate, + "_audio_f32": audio, + } + + with torch.inference_mode(): + for encoded in encode_from_sphn( + mimi, + _iterate_audio(source_audio, sample_interval_size=frame_size, pad=True), + max_batch=1, + ): + for index in range(encoded.shape[-1]): + tokens = generator.step(encoded[:, :, index : index + 1]) + if tokens is None: + continue + decoded_chunks.append(mimi.decode(tokens[:, 1:9])) + other_mimi.decode(tokens[:, 1:9]) + generated_frames += 1 + if generated_frames >= max_frames: + return summary() + return summary() + + return reference_harness.Session( + invoke, + "moshi", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + materialize=_materialize, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run( + list(sys.argv[1:] if argv is None else argv), description=__doc__, load=_load + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml b/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml index 2484898d41..a07bb8bd07 100644 --- a/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml +++ b/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml @@ -9,7 +9,7 @@ accuracy: benchmark: task_output_parity operation: generate request: &request {prompt: Please say hello from Qwen3 Omni in one short sentence., max_new_tokens: 16} - reference: &reference {adapter: hf-qwen3-omni, mode: hf-eager, precision: bf16, output_contract: normalized-text} + reference: &reference {script: tests/benchmark/reference.py, mode: hf-eager, precision: bf16, output_contract: normalized-text} gate: {max_normalized_edit_distance: 0.15} performance: - {name: generate, benchmark: vision_language_performance, operation: generate, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_transformers, output_contract: exact-text}} diff --git a/families/qwen3_omni/tests/benchmark/reference.py b/families/qwen3_omni/tests/benchmark/reference.py new file mode 100644 index 0000000000..e7ad4c51d9 --- /dev/null +++ b/families/qwen3_omni/tests/benchmark/reference.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned Qwen3-Omni text reference.""" + +from __future__ import annotations + +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +SYSTEM_PROMPT = ( + "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " + "capable of perceiving auditory and visual inputs, as well as generating " + "text and speech." +) +CHAT_TEMPLATE = """{%- for message in messages %} +{{- '<|im_start|>' + message.role + '\n' }} +{%- if message.content is string %} +{{- message.content }} +{%- else %} +{%- for item in message.content %} +{%- if item.type == 'text' %}{{- item.text }}{%- endif %} +{%- endfor %} +{%- endif %} +{{- '<|im_end|>\n' }} +{%- endfor %} +{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}""" + + +def _load( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "hf-eager": + raise ValueError("Qwen3-Omni reference requires hf-eager mode") + import torch + from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor + + common = { + "trust_remote_code": arguments.trust_remote_code, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + common["revision"] = arguments.revision + processor = Qwen3OmniMoeProcessor.from_pretrained(arguments.model, **common) + dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[ + arguments.precision + ] + model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( + arguments.model, + torch_dtype=dtype, + device_map=str(options.get("device_map", "cuda:0")), + enable_audio_output=False, + **common, + ).eval() + conversation = [ + {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT}]}, + {"role": "user", "content": [{"type": "text", "text": str(request.get("prompt", ""))}]}, + ] + processor_template = getattr(processor, "chat_template", None) + tokenizer_template = getattr(getattr(processor, "tokenizer", None), "chat_template", None) + template = ( + processor_template + if isinstance(processor_template, str) and processor_template.strip() + else tokenizer_template + if isinstance(tokenizer_template, str) and tokenizer_template.strip() + else CHAT_TEMPLATE + ) + inputs = processor.apply_chat_template( + conversation, + chat_template=template, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding=True, + ).to(model.device) + max_new_tokens = int(request.get("max_new_tokens", 16)) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + text_ids = model.generate( + **inputs, + thinker_max_new_tokens=max_new_tokens, + thinker_do_sample=False, + return_audio=False, + ) + generated = text_ids[:, int(inputs["input_ids"].shape[-1]) :] + token_ids = [int(token) for token in generated[0].detach().cpu().tolist()] + return { + "text": processor.batch_decode(generated, skip_special_tokens=True)[0].strip(), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run( + list(sys.argv[1:] if argv is None else argv), description=__doc__, load=_load + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/qwen3_omni/tests/test_benchmark_reference.py b/families/qwen3_omni/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..664a81a9bd --- /dev/null +++ b/families/qwen3_omni/tests/test_benchmark_reference.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_text_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/qwen3-omni-30b-a3b-instruct.yaml").read_text(encoding="utf-8") + ) + case = profile["performance"][0] + assert case["operation"] == "generate" + assert case["request"]["max_new_tokens"] == 16 + assert case["reference"]["script"] == "tests/benchmark/reference.py" + assert "adapter" not in case["reference"] + + source = (ROOT / "benchmark/reference.py").read_text(encoding="utf-8") + assert "enable_audio_output=False" in source + assert "return_audio=False" in source diff --git a/families/sana_wm/tests/benchmark/reference.py b/families/sana_wm/tests/benchmark/reference.py new file mode 100644 index 0000000000..3b46126303 --- /dev/null +++ b/families/sana_wm/tests/benchmark/reference.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned SANA-WM reference.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _execute( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Measurement: + options = dict(options) + testcase_name = getattr(arguments, "testcase_name", None) + if testcase_name is not None: + manifest_value = json.loads(arguments.manifest.read_text(encoding="utf-8")) + matches = [ + value + for value in manifest_value.get("testcases", []) + if isinstance(value, Mapping) and value.get("name") == testcase_name + ] + if len(matches) != 1: + raise ValueError(f"SANA-WM requires exactly one testcase named {testcase_name!r}") + for name in ( + "translation_speed", + "rotation_speed_deg", + "fps", + "flow_shift", + "no_action_overlay", + ): + if name in matches[0]: + options.setdefault(name, matches[0][name]) + reference_repo = Path(str(options.get("reference_repo", ""))).resolve() + if not reference_repo.is_dir(): + raise ValueError("SANA-WM reference requires reference_repo") + model_root = arguments.manifest.resolve().parent.parent + + def model_input(value: Any) -> Path: + path = Path(str(value)) + return path.resolve() if path.is_absolute() else (model_root / path).resolve() + + image = model_input(request["image_path"]) + intrinsics = model_input( + options.get("intrinsics_path", options.get("intrinsics", "assets/demo_0_intrinsics.npy")) + ) + for label, path in (("image", image), ("intrinsics", intrinsics)): + if not path.is_file(): + raise FileNotFoundError(f"SANA-WM {label} input does not exist: {path}") + prompt_text = str(request.get("prompt", "")) + if not prompt_text: + raise ValueError("SANA-WM request requires prompt") + action = request.get("action") + if not isinstance(action, str) or not action: + raise ValueError("SANA-WM request requires a non-empty action") + if "model_dir" not in options: + raise ValueError("SANA-WM reference requires model_dir") + model_dir = Path(str(options["model_dir"])).resolve() + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + num_frames = int( + request["num_frames"] + if "num_frames" in request + else manifest["video_num_frames"] + ) + with tempfile.TemporaryDirectory(prefix="trtmc-perf-sana-wm-") as temporary: + root = Path(temporary) + output = root / "benchmark.json" + prompt = root / "prompt.txt" + prompt.write_text(prompt_text, encoding="utf-8") + command = [ + sys.executable, + str(Path(__file__).with_name("upstream_reference.py")), + "--reference-repo", + str(reference_repo), + "--image", + str(image), + "--model-dir", + str(model_dir), + "--prompt", + str(prompt), + "--action", + action, + "--intrinsics", + str(intrinsics), + "--translation_speed", + str( + request["translation_speed"] + if "translation_speed" in request + else options["translation_speed"] + ), + "--rotation_speed_deg", + str( + request["rotation_speed_deg"] + if "rotation_speed_deg" in request + else options["rotation_speed_deg"] + ), + "--num_frames", + str(num_frames), + "--fps", + str(request["fps"] if "fps" in request else options["fps"]), + "--step", + str(request["num_steps"]), + "--cfg_scale", + str(request["cfg_scale"]), + "--flow_shift", + str( + request["flow_shift"] if "flow_shift" in request else options["flow_shift"] + ), + "--seed", + str(request["seed"]), + "--refiner_seed", + str(request["seed"]), + "--warmup", + str(arguments.warmup), + "--iterations", + str(arguments.iterations), + "--output", + str(output), + ] + no_overlay = ( + request["no_action_overlay"] + if "no_action_overlay" in request + else options["no_action_overlay"] + ) + if bool(no_overlay): + command.append("--no_action_overlay") + completed = subprocess.run( + command, + cwd=reference_repo, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + raise RuntimeError( + f"SANA-WM reference failed with rc={completed.returncode}: " + f"{completed.stderr[-4000:]}" + ) + payload = json.loads(output.read_text(encoding="utf-8")) + summary = dict(payload.get("output_summary", {})) + artifacts = summary.get("frame_artifacts", []) + if artifacts: + destination = arguments.output.with_suffix(".media").resolve() + destination.mkdir(parents=True, exist_ok=True) + copied = [] + for source_value in artifacts: + source = Path(str(source_value)) + target = destination / source.name + shutil.copyfile(source, target) + copied.append(str(target)) + summary["frame_artifacts"] = copied + return reference_harness.Measurement( + [float(value) for value in payload.get("samples_ms", [])], + summary, + "sana-wm-pytorch", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=False, + ) + + +def _run_compat( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: + """Compatibility shape for family-local unit tests written before the move.""" + value = _execute(arguments, request, options) + return ( + list(value.samples_ms), + dict(value.output_summary), + value.backend, + value.timing_scope, + value.input_preparation_included, + value.asset_loading_included, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run_premeasured( + list(sys.argv[1:] if argv is None else argv), + description=__doc__, + execute=_execute, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml index a061e206a2..d7c4f20570 100644 --- a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml +++ b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml @@ -27,7 +27,7 @@ accuracy: fps: 16 flow_shift: 9.8 no_action_overlay: true - reference: &reference {adapter: upstream-sana-wm, mode: pytorch-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {intrinsics_path: ../assets/demo_0_intrinsics.npy}} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {intrinsics_path: ../assets/demo_0_intrinsics.npy}} gate: {min_psnr: 5.0, min_ssim: 0.1} performance: - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: official_sana, output_contract: media-shape}} diff --git a/apps/benchmark/performance/baselines/sana_wm_reference.py b/families/sana_wm/tests/benchmark/upstream_reference.py similarity index 100% rename from apps/benchmark/performance/baselines/sana_wm_reference.py rename to families/sana_wm/tests/benchmark/upstream_reference.py diff --git a/families/sana_wm/tests/test_benchmark_reference.py b/families/sana_wm/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..c78dffef59 --- /dev/null +++ b/families/sana_wm/tests/test_benchmark_reference.py @@ -0,0 +1,662 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace +import json +from pathlib import Path +import sys +from types import ModuleType, SimpleNamespace + +import numpy as np +import pytest +import yaml + +from families.sana_wm.tests.benchmark import reference as sana_family_reference +from families.sana_wm.tests.benchmark import upstream_reference as sana_wm_reference +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references import generic_reference + + +REPO = Path(__file__).resolve().parents[3] +SUITE = REPO / "qualification_tests/benchmark_qualification/performance/config/release.yaml" + + +def _environment(tmp_path: Path) -> tuple[Path, perf.Environment]: + tools = tmp_path / "tools" + tools.mkdir() + for name in ("bench", "worker", "hf.py", "task.py"): + path = tools / name + path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + path.chmod(0o755) + runtime = tmp_path / "runtime" + runtime.mkdir() + for name in ( + "libtrtmc_runtime.so", + "libtrtmc_backend_trt.so", + "libtrtmc_model_gpt2.so", + "libtrtmc_model_lance.so", + ): + (runtime / name).write_bytes(b"") + _, entries, _ = perf._load_suite_file(SUITE) + reference_fields = { + str(declaration["environment"]) + for entry in entries + for declaration in entry.get("baseline", {}).get("reference_inputs", {}).values() + } + references = {} + for name in reference_fields: + path = tmp_path / name + path.mkdir() + references[name] = str(path) + value = { + "schema_version": perf.ENVIRONMENT_SCHEMA, + "name": "test", + "tools": { + "trtmc_bench": str(tools / "bench"), + "trtmc_worker": str(tools / "worker"), + "hf_transformers_runner": str(tools / "hf.py"), + "task_reference_runner": str(tools / "task.py"), + }, + "references": references, + "storage": { + "results_root": str(tmp_path / "results"), + "scratch_root": str(tmp_path / "scratch"), + "bundle_cache": str(tmp_path / "bundles"), + "bundle_roots": [], + "runtime_root": str(runtime), + "bundle_retention": "retain", + }, + "execution": {"local_files_only": True, "timeout_seconds": 10}, + } + path = tmp_path / "environment.yaml" + path.write_text(yaml.safe_dump(value), encoding="utf-8") + return path, perf.load_environment(path) + + +def test_sana_reference_reports_materialized_video_shape() -> None: + video = np.stack( + [ + np.zeros((24, 32, 3), dtype=np.uint8), + np.ones((24, 32, 3), dtype=np.uint8), + ] + ) + summary = sana_wm_reference.media_summary(video) + assert summary == { + "media_type": "video", + "media_count": 2, + "num_frames": 2, + "height": 24, + "width": 32, + "channels": 3, + } + assert perf._media_shape(summary) == (2, 24, 32, 3) + source = (REPO / "families/sana_wm/tests/benchmark/reference.py").read_text( + encoding="utf-8" + ) + assert 'request.get("action"' in source + + +def test_sana_world_request_preserves_official_camera_controls(tmp_path: Path) -> None: + model = replace( + perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional"), + task="world_model_generation", + ) + request = perf.resolve_case( + model, tmp_path / "model.bundle", selected_task="world_model_generation", + ).request + assert request["translation_speed"] == 0.055 + assert request["rotation_speed_deg"] == 1.2 + assert request["fps"] == 16 + assert request["flow_shift"] == 9.8 + assert request["no_action_overlay"] is True + + +@pytest.mark.parametrize("route", ["builtin", "script", "hf"]) +def test_reference_testcase_name_is_only_sent_to_builtin_runner(monkeypatch, tmp_path, route): + _, environment = _environment(tmp_path) + # Exercise explicit runner fixtures, not a family's active owned replacement. + _, entries, _ = perf._load_suite_file(SUITE) + entry = perf.resolve_entries( + [value for value in entries if value["id"] == "sana_wm.generate_image"], environment, + )[0] + baseline = dict(entry.spec["baseline"]) + if route == "builtin": + baseline.pop("script") + baseline["adapter"] = "hf-diffusers" + elif route == "script": + baseline["script"] = "tests/reference.py" + monkeypatch.setattr(perf, "_family_script", lambda _: tmp_path / "reference.py") + elif route == "hf": + baseline["runner"] = "hf-transformers" + entry = replace(entry, spec={**entry.spec, "baseline": baseline}) + command = perf.baseline_command(entry, environment, tmp_path / "reference.json") + assert command[command.index("--case-name") + 1] == entry.spec["id"] + if route == "builtin": + assert command[command.index("--testcase-name") + 1] == entry.case.testcase_name + parser = generic_reference.build_parser() + parsed = parser.parse_args(command[2:]) + assert parsed.testcase_name == entry.case.testcase_name + index = command.index("--testcase-name") + direct = command[2:index] + command[index + 2:] + assert parser.parse_args(direct).testcase_name is None + else: + assert "--testcase-name" not in command + + +@pytest.mark.parametrize("selection", ["selected", "missing", "duplicate", ""]) +def test_sana_reference_resolves_only_exact_testcase_before_execution( + monkeypatch, tmp_path, selection, +): + manifest = tmp_path / "manifest.json" + controls = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + chosen = {"name": "selected", **controls} + manifest.write_text(json.dumps({"video_num_frames": 321, "testcases": [ + {"name": "performance-entry", **dict.fromkeys(controls, "wrong testcase")}, + chosen, *([chosen] if selection == "duplicate" else []), + ]})) + arguments = SimpleNamespace( + manifest=manifest, testcase_name="selected" if selection == "duplicate" else selection, + case_name="performance-entry", warmup=1, iterations=2, + ) + assets = REPO / "families/sana_wm/tests/assets" + request = {"prompt": "drive forward", "image_path": str(assets / "demo_0.png"), + "action": "w-320", "num_steps": 60, "cfg_scale": 5.0, "seed": 42} + options = {"reference_repo": str(tmp_path), "model_dir": str(tmp_path / "model"), + "intrinsics": str(assets / "demo_0_intrinsics.npy")} + before = deepcopy(request), deepcopy(options), manifest.read_bytes() + + def run(command, **kwargs): + assert selection == "selected", "invalid selection must fail before reference execution" + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(controls[name]) + assert "--no_action_overlay" in command + assert command[command.index("--num_frames") + 1] == "321" + raise RuntimeError("captured selected reference command") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + if selection == "selected": + with pytest.raises(RuntimeError, match="captured selected reference command"): + sana_family_reference._run_compat(arguments, request, options) + else: + with pytest.raises(ValueError, match="exactly one testcase"): + sana_family_reference._run_compat(arguments, request, options) + assert (request, options, manifest.read_bytes()) == before + + +def test_sana_reference_options_use_resolved_testcase_and_explicit_options( + monkeypatch, tmp_path: Path, +) -> None: + _, environment = _environment(tmp_path) + # This tests the existing builtin adapter even after an owned script takes over. + _, entries, _ = perf._load_suite_file(SUITE) + selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] + entry = perf.resolve_entries(selected, environment)[0] + original = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + assert entry.case.testcase_name != entry.spec["id"] + testcase = next(value for value in entry.manifest["testcases"] + if value["name"] == entry.case.testcase_name) + other = {**testcase, "name": entry.spec["id"], **dict.fromkeys(original, "wrong testcase")} + entry = replace(entry, manifest={**entry.manifest, "testcases": [other, testcase]}) + semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") + request = generic_reference.flatten_config(semantic.request) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(entry.manifest)) + arguments = SimpleNamespace( + manifest=manifest_path, testcase_name=entry.case.testcase_name, warmup=1, iterations=2, + ) + before = json.dumps(entry.manifest, sort_keys=True) + options = perf._adapter_options(entry, environment) + assert {name: options[name] for name in original} == original + assert "action" not in options and "prompt" not in options + assert options["reference_repo"] == str(Path(environment.references["sana_repo"]).resolve()) + assert options["model_dir"] == str(Path(environment.references["sana_model"]).resolve()) + options["intrinsics"] = str(entry.model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy") + expected = original + + def run(command, **kwargs): + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(expected[name]) + assert ("--no_action_overlay" in command) is expected["no_action_overlay"] + assert command[command.index("--action") + 1] == request["action"] + assert Path(command[command.index("--prompt") + 1]).read_text() == request["prompt"] + raise RuntimeError("captured reference options") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + original_options = deepcopy(options) + with pytest.raises(RuntimeError, match="captured reference options"): + sana_family_reference._run_compat(arguments, request, options) + assert options == original_options + explicit = {"translation_speed": 0.0, "rotation_speed_deg": 0.0, + "fps": 0, "flow_shift": 0.0, "no_action_overlay": False} + configured = {**entry.spec["baseline"].get("adapter_options", {}), **explicit} + entry = replace(entry, spec={**entry.spec, "baseline": {**entry.spec["baseline"], "adapter_options": configured}}) + options = perf._adapter_options(entry, environment) + assert {name: options[name] for name in original} == explicit + assert options["no_action_overlay"] is False + options["intrinsics"] = original_options["intrinsics"] + explicit_options = deepcopy(options) + expected = explicit + with pytest.raises(RuntimeError, match="captured reference options"): + sana_family_reference._run_compat(arguments, request, options) + assert options == explicit_options + assert request == generic_reference.flatten_config(semantic.request) + assert json.loads(manifest_path.read_text()) == entry.manifest + assert json.dumps(entry.manifest, sort_keys=True) == before + assert entry.spec["baseline"]["adapter_options"] == configured + + +def test_sana_semantic_request_metadata_moves_only_to_reference_options( + monkeypatch, tmp_path: Path, +) -> None: + _, environment = _environment(tmp_path) + # Keep the builtin reference contract independent of canonical script selection. + _, entries, _ = perf._load_suite_file(SUITE) + selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] + entry = perf.resolve_entries(selected, environment)[0] + semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") + entry = replace(entry, case=semantic) + options = perf._adapter_options(entry, environment) + expected = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + assert not expected.keys() & semantic.request.keys() + assert {name: options[name] for name in expected} == expected + command = perf.baseline_command(entry, environment, tmp_path / "reference.json") + arguments = reference_harness.parser("test").parse_args(command[2:]) + + def run(command, **kwargs): + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(expected[name]) + assert "--no_action_overlay" in command + raise RuntimeError("captured reference metadata") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + with pytest.raises(RuntimeError, match="captured reference metadata"): + sana_family_reference._run_compat( + arguments, generic_reference.flatten_config(semantic.request), options + ) + assert semantic.request["action"] == entry.manifest["testcases"][0]["action"] + assert semantic.request["seed"] == 42 and semantic.request["num_steps"] == 60 + assert semantic.request["cfg_scale"] == 5.0 + + +def test_sana_reference_calls_official_pipeline_with_exact_workload( + monkeypatch, tmp_path: Path +) -> None: + captured: dict[str, object] = {"generate": []} + reference_repo = tmp_path / "Sana" + reference_repo.mkdir() + model_dir = tmp_path / "model" + (model_dir / "dit").mkdir(parents=True) + (model_dir / "refiner/text_encoder").mkdir(parents=True) + (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") + (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") + image = tmp_path / "image.png" + image.write_bytes(b"image") + prompt = tmp_path / "prompt.txt" + prompt.write_text("drive forward", encoding="utf-8") + intrinsics = tmp_path / "intrinsics.npy" + intrinsics.write_bytes(b"intrinsics") + output = tmp_path / "result.json" + + class FakeImage: + def convert(self, mode): + captured["image_mode"] = mode + return self + + def save(self, path): + Path(path).write_bytes(b"png") + return None + + pil = ModuleType("PIL") + pil.Image = SimpleNamespace(open=lambda path: FakeImage(), fromarray=lambda value: FakeImage()) + monkeypatch.setitem(sys.modules, "PIL", pil) + + synchronize_calls = [] + torch = ModuleType("torch") + torch.cuda = SimpleNamespace( + is_available=lambda: True, + synchronize=lambda: synchronize_calls.append(True), + ) + monkeypatch.setitem(sys.modules, "torch", torch) + + pyrallis = ModuleType("pyrallis") + + def parse_config(**kwargs): + captured["parse"] = kwargs + return "config" + + pyrallis.parse = parse_config + monkeypatch.setitem(sys.modules, "pyrallis", pyrallis) + + class RefinerSettings: + def __init__(self, **kwargs): + captured["refiner"] = kwargs + + class GenerationParams: + def __init__(self, **kwargs): + captured["generation"] = kwargs + + class Pipeline: + def __init__(self, **kwargs): + captured["pipeline"] = kwargs + + def generate(self, *args): + captured["generate"].append(args) + return { + "video": np.zeros((321, 24, 32, 3), dtype=np.uint8), + "c2w": "camera", + } + + trajectory = np.zeros((321, 4, 4), dtype=np.float32) + official = SimpleNamespace( + InferenceConfig=object, + RefinerSettings=RefinerSettings, + GenerationParams=GenerationParams, + SanaWMPipeline=Pipeline, + action_string_to_c2w=lambda action, **kwargs: ( + captured.update(action=(action, kwargs)) or trajectory + ), + _snap_num_frames=lambda value, **kwargs: value, + resize_and_center_crop=lambda value: ("cropped", (1, 1), (2, 2), (0, 0)), + load_intrinsics=lambda path, frames: ( + captured.update(load_intrinsics=(path, frames)) or "raw-intrinsics" + ), + transform_intrinsics_for_crop=lambda value, *sizes: ( + captured.update(transform_intrinsics=(value, sizes)) or "intrinsics" + ), + apply_overlay=lambda *_: (_ for _ in ()).throw( + AssertionError("no-action-overlay must skip overlay") + ), + ) + monkeypatch.setattr(sana_wm_reference, "_official_module", lambda path: official) + monkeypatch.setattr( + sys, + "argv", + [ + "sana_wm_reference.py", + "--reference-repo", + str(reference_repo), + "--image", + str(image), + "--model-dir", + str(model_dir), + "--prompt", + str(prompt), + "--action", + "w-320", + "--intrinsics", + str(intrinsics), + "--num_frames", + "321", + "--fps", + "16", + "--step", + "60", + "--cfg_scale", + "5.0", + "--flow_shift", + "9.8", + "--seed", + "42", + "--refiner_seed", + "42", + "--translation_speed", + "0.055", + "--rotation_speed_deg", + "1.2", + "--no_action_overlay", + "--warmup", + "1", + "--iterations", + "2", + "--output", + str(output), + ], + ) + + assert sana_wm_reference.main() == 0 + assert captured["action"] == ( + "w-320", + {"translation_speed": 0.055, "rotation_speed_deg": 1.2}, + ) + assert captured["refiner"] == { + "root": model_dir / "refiner", + "gemma_root": model_dir / "refiner/text_encoder", + "seed": 42, + } + assert captured["generation"] == { + "num_frames": 321, + "fps": 16, + "step": 60, + "cfg_scale": 5.0, + "flow_shift": 9.8, + "seed": 42, + } + assert len(captured["generate"]) == 3 + assert len(synchronize_calls) == 5 + result = json.loads(output.read_text(encoding="utf-8")) + assert result["output_summary"]["num_frames"] == 321 + + +def test_sana_reference_requires_action_from_current_request(tmp_path: Path) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json" + ) + with pytest.raises(ValueError, match="non-empty action"): + sana_family_reference._run_compat( + arguments, + {"prompt": "drive", "image_path": "assets/demo_0.png"}, + {"reference_repo": str(checkout)}, + ) + + +@pytest.mark.parametrize("frame_controls", [{"num_frames": 321}, {}]) +def test_sana_generic_reference_uses_one_explicit_official_command( + monkeypatch, tmp_path: Path, frame_controls: dict, +) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + model_dir = tmp_path / "model" + model_dir.mkdir() + captured = {} + + def run(command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples_ms": [1.0, 2.0], + "output_summary": { + "media_type": "video", + "media_count": 321, + "num_frames": 321, + "height": 704, + "width": 1280, + "channels": 3, + }, + } + ), + encoding="utf-8", + ) + return SimpleNamespace(returncode=0, stderr="") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", + warmup=1, + iterations=2, + ) + result = sana_family_reference._run_compat( + arguments, + { + "prompt": "drive forward", + "image_path": "assets/demo_0.png", + "action": "w-80,jw-40,w-40,lw-60,w-100", + "translation_speed": 0.055, + "rotation_speed_deg": 1.2, + **frame_controls, + "fps": 16, + "num_steps": 60, + "cfg_scale": 5.0, + "flow_shift": 9.8, + "seed": 42, + "no_action_overlay": True, + }, + { + "reference_repo": str(checkout), + "model_dir": str(model_dir), + "intrinsics": "assets/demo_0_intrinsics.npy", + }, + ) + + command = captured["command"] + assert command[command.index("--reference-repo") + 1] == str(checkout.resolve()) + assert command[command.index("--translation_speed") + 1] == "0.055" + assert command[command.index("--rotation_speed_deg") + 1] == "1.2" + assert command[command.index("--num_frames") + 1] == "321" + assert command[command.index("--fps") + 1] == "16" + assert command[command.index("--flow_shift") + 1] == "9.8" + assert command[command.index("--refiner_seed") + 1] == "42" + assert command[command.index("--warmup") + 1] == "1" + assert command[command.index("--iterations") + 1] == "2" + assert "--no_action_overlay" in command + assert "env" not in captured["kwargs"] + assert result[0] == [1.0, 2.0] + assert result[1]["num_frames"] == 321 + + +@pytest.mark.parametrize("controls,manifest_frames,expected", [ + ({}, 321, "321"), + ({"num_frames": 7}, 321, "7"), + ({"num_frames": 0}, 321, "0"), + ({"config": {"num_frames": 0}}, 321, "0"), + ({"num_frames": 7}, None, "7"), + ({}, None, None), +]) +def test_sana_semantic_reference_uses_manifest_frames_only_when_absent( + monkeypatch, tmp_path: Path, controls: dict, manifest_frames: int | None, expected: str | None, +) -> None: + model = perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional") + model = replace(model, testcases=({**model.testcases[0], **controls},)) + case = perf.resolve_case(model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") + request = generic_reference.flatten_config(case.request) + before = dict(request) + manifest = json.loads(model.manifest_path.read_text()) + if manifest_frames is None: + del manifest["video_num_frames"] + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest)) + checkout = tmp_path / "Sana" + checkout.mkdir() + options = { + "reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), + "intrinsics": str(model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy"), + } + testcase = next(value for value in manifest["testcases"] if value["name"] == case.testcase_name) + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay"): + options[name] = testcase[name] + assert name not in request + + def run(command, **kwargs): + assert command[command.index("--num_frames") + 1] == expected + assert command[command.index("--action") + 1] == request["action"] + assert command[command.index("--translation_speed") + 1] == "0.055" + assert command[command.index("--rotation_speed_deg") + 1] == "1.2" + assert command[command.index("--fps") + 1] == "16" + assert command[command.index("--flow_shift") + 1] == "9.8" + assert "--no_action_overlay" in command + raise RuntimeError("captured reference command") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + arguments = SimpleNamespace(manifest=manifest_path, warmup=1, iterations=2) + if expected is None: + with pytest.raises(KeyError, match="video_num_frames"): + sana_family_reference._run_compat(arguments, request, options) + else: + with pytest.raises(RuntimeError, match="captured reference command"): + sana_family_reference._run_compat(arguments, request, options) + assert request == before + + +@pytest.mark.parametrize("source", ["request", "config", "options"]) +@pytest.mark.parametrize("controls", [ + {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True}, + {"translation_speed": 0.0, "rotation_speed_deg": 0.0, + "fps": 0, "flow_shift": 0.0, "no_action_overlay": False}, +]) +def test_sana_reference_control_presence_and_request_precedence( + monkeypatch, tmp_path: Path, source: str, controls: dict, +) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", + warmup=1, iterations=2, + ) + request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", + "action": "w-80,jw-40,w-40,lw-60,w-100", "num_frames": 321, + "num_steps": 60, "cfg_scale": 5.0, "seed": 42} + options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), + "translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + if source == "options": + options.update(controls) + elif source == "config": + request = generic_reference.flatten_config({**request, "config": controls}) + else: + request.update(controls) + before = dict(request), dict(options) + + def run(command, **kwargs): + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(controls[name]) + assert ("--no_action_overlay" in command) is controls["no_action_overlay"] + assert command[command.index("--action") + 1] == request["action"] + assert command[command.index("--refiner_seed") + 1] == "42" + assert "env" not in kwargs + raise RuntimeError("captured reference controls") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + with pytest.raises(RuntimeError, match="captured reference controls"): + sana_family_reference._run_compat(arguments, request, options) + assert (request, options) == before + + +@pytest.mark.parametrize("missing", [ + "translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay", +]) +def test_sana_reference_control_missing_from_request_and_options_still_errors( + monkeypatch, tmp_path: Path, missing: str, +) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", + warmup=1, iterations=2, + ) + request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", "action": "w-320", + "num_frames": 321, "num_steps": 60, "cfg_scale": 5.0, "seed": 42} + options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), + "translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + del options[missing] + monkeypatch.setattr( + sana_family_reference.subprocess, + "run", + lambda *args, **kwargs: pytest.fail( + "missing control must fail before reference execution" + ), + ) + with pytest.raises(KeyError, match=missing): + sana_family_reference._run_compat(arguments, request, options) diff --git a/families/segformer/tests/benchmark/reference.py b/families/segformer/tests/benchmark/reference.py index efa3734065..f8f637bcfd 100644 --- a/families/segformer/tests/benchmark/reference.py +++ b/families/segformer/tests/benchmark/reference.py @@ -9,7 +9,10 @@ import argparse import json from pathlib import Path -from typing import Any, Sequence +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness def _parser() -> argparse.ArgumentParser: @@ -26,7 +29,7 @@ def _dtype(torch: Any, precision: str) -> Any: raise ValueError(f"unsupported reference precision {precision!r}") from error -def main(argv: Sequence[str] | None = None) -> int: +def _accuracy(argv: Sequence[str] | None = None) -> int: arguments = _parser().parse_args(argv) request = json.loads(arguments.request.read_text(encoding="utf-8")) samples = request.get("samples") @@ -78,5 +81,59 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 +def _performance_session( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForSemanticSegmentation + + if arguments.mode != "hf-eager": + raise ValueError("SegFormer reference requires hf-eager mode") + options = {"revision": arguments.revision} if arguments.revision else {} + processor = AutoImageProcessor.from_pretrained(arguments.model, use_fast=False, **options) + model = ( + AutoModelForSemanticSegmentation.from_pretrained( + arguments.model, + torch_dtype=_dtype(torch, arguments.precision), + **options, + ) + .eval() + .to("cuda") + ) + image = Image.open(str(request["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(image.height, image.width)] + )[0] + return { + "num_masks": 1, + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/families/segformer/tests/benchmark/segformer-b0-ade.yaml b/families/segformer/tests/benchmark/segformer-b0-ade.yaml index 60bcd4300c..eb3b8d8382 100644 --- a/families/segformer/tests/benchmark/segformer-b0-ade.yaml +++ b/families/segformer/tests/benchmark/segformer-b0-ade.yaml @@ -39,8 +39,7 @@ performance: iterations: 10 reference: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: semantic-segmentation} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers precision: fp32 diff --git a/families/whisper/tests/benchmark/reference.py b/families/whisper/tests/benchmark/reference.py index bcc3d0aef4..27794eacd6 100644 --- a/families/whisper/tests/benchmark/reference.py +++ b/families/whisper/tests/benchmark/reference.py @@ -8,7 +8,7 @@ from typing import Sequence -from tools.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.references import speech_io def main(argv: Sequence[str] | None = None) -> int: diff --git a/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md b/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md index a5b971b443..f45783eb40 100644 --- a/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md +++ b/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md @@ -45,7 +45,7 @@ trtmc-bench run --model --case \ --output ``` -Use `tools/perf_matrix.py` when the claim targets a checked-in release entry. +Use `python3 -m qualification_tests.benchmark_qualification.performance` when the claim targets a checked-in release entry. Report every attempt in a compact table, including effective request values, correctness, measured objective, confounders, and unrun targets. It is valid for the original configuration to remain the best qualified result. diff --git a/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md b/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md index e9fba4f0cd..b57ce684b9 100644 --- a/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md +++ b/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md @@ -10,7 +10,7 @@ description: >- Choose the evidence level before running: - For one model or testcase, use `trtmc-bench`. -- For a checked-in release comparison, use `tools/perf_matrix.py`. +- For a checked-in release comparison, use `python3 -m qualification_tests.benchmark_qualification.performance`. The public benchmark boundary is `public_task_call_wall`. It excludes bundle building, process startup, warmup, report generation, and bundle loading. @@ -39,9 +39,9 @@ confounder instead of one causal speedup percentage. For release evidence, first check and then run the exact entry: ```bash -python3 tools/perf_matrix.py check \ +python3 -m qualification_tests.benchmark_qualification.performance check \ --environment --entry -python3 tools/perf_matrix.py run \ +python3 -m qualification_tests.benchmark_qualification.performance run \ --environment --entry ``` diff --git a/pyproject.toml b/pyproject.toml index cee00e7423..34f1d1ee47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,7 @@ include = [ "requirements", "third_party", "examples", - "tools/perf_matrix.py", + "qualification_tests/benchmark_qualification", ] exclude = [ "__pycache__", @@ -95,6 +95,7 @@ testpaths = [ "examples/models/cosmos3/dual_spark/test_cosmos3_dual_spark_source.py", "examples/models/nemotron_voicechat/full_duplex/test_voicechat_full_duplex_source.py", "tools/tests", + "qualification_tests/benchmark_qualification", ] markers = [ "gpu: tests requiring a CUDA GPU", diff --git a/tools/benchmark_qualification/__init__.py b/qualification_tests/__init__.py similarity index 65% rename from tools/benchmark_qualification/__init__.py rename to qualification_tests/__init__.py index 8ffcd736c3..826b2d60d4 100644 --- a/tools/benchmark_qualification/__init__.py +++ b/qualification_tests/__init__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Repository-only Accuracy and Performance qualification automation.""" +"""Repository-owned validation and qualification modules.""" diff --git a/qualification_tests/benchmark_qualification/__init__.py b/qualification_tests/benchmark_qualification/__init__.py new file mode 100644 index 0000000000..efc4c49e8a --- /dev/null +++ b/qualification_tests/benchmark_qualification/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Internal model qualification framework.""" diff --git a/tools/benchmark_qualification/accuracy.py b/qualification_tests/benchmark_qualification/accuracy.py similarity index 98% rename from tools/benchmark_qualification/accuracy.py rename to qualification_tests/benchmark_qualification/accuracy.py index e88d5dd2fe..243dce20f0 100644 --- a/tools/benchmark_qualification/accuracy.py +++ b/qualification_tests/benchmark_qualification/accuracy.py @@ -27,12 +27,12 @@ robot_action_metrics, robot_action_passes, ) +from qualification_tests.benchmark_qualification.performance import reference_protocol from .catalog import QualificationCase, QualificationError, load_benchmark from .datasets import Dataset, resolve_dataset from .runtime import ( RuntimeContext, - benchmark_executable, prepare_bundle, reference_environment_options, reference_python, @@ -107,11 +107,24 @@ def _task_output_parity( request = _resolve_task_assets(case, request) descriptor = write_model_descriptor(case, output, request, context=context) descriptor_value = json.loads(descriptor.read_text(encoding="utf-8")) - adapter = str(reference.get("adapter", "")) - if not adapter: - raise QualificationError("task-output reference.adapter must be set") - adapter_options = dict(reference.get("adapter_options", {})) + adapter_value = reference.get("adapter") + script_value = reference.get("script") family_root = case.source.parents[2].resolve() + if (adapter_value is None) == (script_value is None): + raise QualificationError("task-output reference requires exactly one adapter or script") + adapter = str(adapter_value) if adapter_value is not None else None + script = None + if script_value is not None: + relative = Path(str(script_value)) + script = (family_root / relative).resolve() + if ( + relative.is_absolute() + or ".." in relative.parts + or not script.is_relative_to(family_root) + or not script.is_file() + ): + raise QualificationError(f"task-output reference script is unavailable: {script_value}") + adapter_options = dict(reference.get("adapter_options", {})) for name, value in tuple(adapter_options.items()): if not name.endswith("_path") or not isinstance(value, str): continue @@ -123,48 +136,38 @@ def _task_output_parity( adapter_options[name] = str(path) adapter_options = reference_environment_options(case, context, adapter_options) reference_path = output / "reference-result.json" - command = [ - str(reference_python(case, context)), - str(context.repository / "apps/benchmark/performance/baselines/task_reference.py"), - "--adapter", - adapter, - "--family", - case.family, - "--operation", - str(configured["operation"]), - "--model", - str(descriptor_value["hf_id"]), - "--manifest", - str(descriptor), - "--testcase-name", - case.name, - "--request-json", - json.dumps(request), - "--adapter-options-json", - json.dumps(adapter_options), - "--precision", - str(reference.get("precision", case.candidate["precision"])), - "--mode", - str(reference.get("mode", "hf-eager")), - "--warmup", - "0", - "--iterations", - "1", - "--case-name", - case.name, - "--output", - str(reference_path), - ] revision = reference.get("revision", descriptor_value.get("hf_revision")) - if revision: - command.extend(("--revision", str(revision))) selected_task = case.candidate.get("selected_task") - if selected_task: - command.extend(("--selected-task", str(selected_task))) - if bool(case.candidate.get("trust_remote_code", False)): - command.append("--trust-remote-code") - if os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1": - command.append("--local-files-only") + command = reference_protocol.command( + python=reference_python(case, context), + generic_runner=( + context.repository + / "qualification_tests/benchmark_qualification/performance/references/generic_reference.py" + ), + script=script, + family=case.family, + operation=str(configured["operation"]), + manifest=descriptor, + selected_task=str(selected_task or case.candidate["task"]) if script else ( + str(selected_task) if selected_task else None + ), + testcase_name=case.name, + adapter=adapter, + adapter_options=adapter_options, + timing_contract={}, + padding="longest", + model=str(descriptor_value["hf_id"]), + revision=str(revision) if revision else None, + request=request, + precision=str(reference.get("precision", case.candidate["precision"])), + mode=str(reference.get("mode", "hf-eager")), + warmup=0, + iterations=1, + case_name=case.name, + output=reference_path, + trust_remote_code=bool(case.candidate.get("trust_remote_code", False)), + local_files_only=os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + ) completed = run_command(command, output, "reference", timeout=7200, verbose=context.verbose) if completed.returncode != 0: raise QualificationError(f"task-output Accuracy reference failed; see {output}") @@ -211,7 +214,11 @@ def _task_output_parity( "model": case.model, "benchmark": case.benchmark, "bundle": str(bundle), - "reference": {"adapter": adapter, "model": reference_result.get("model")}, + "reference": { + "adapter": adapter, + "script": str(script) if script else None, + "model": reference_result.get("model"), + }, "metrics": metrics, "gate": dict(gate), } @@ -690,7 +697,7 @@ def _text_generation_parity( runner_value = reference.get("command") if runner_value is None: runner = ( - context.repository / "tools/benchmark_qualification/references/hf_text_generation.py" + context.repository / "qualification_tests/benchmark_qualification/references/hf_text_generation.py" ) else: if not isinstance(runner_value, str) or not runner_value: @@ -816,7 +823,7 @@ def _encoder_embedding_parity( reference_path = output / "reference.json" output.mkdir(parents=True, exist_ok=True) _json(request_path, reference_request) - runner = context.repository / "tools/benchmark_qualification/references/hf_encoder.py" + runner = context.repository / "qualification_tests/benchmark_qualification/references/hf_encoder.py" completed = run_command( [ str(reference_python(case, context)), @@ -1397,7 +1404,7 @@ def _candidate_outputs( _json(data_path, list(requests)) candidate_output = output / "candidate-run" command = [ - str(benchmark_executable(case, context)), + str(context.trtmc_bench), "run", "--model", str(descriptor), @@ -2677,7 +2684,7 @@ def _image_classification_parity( reference_path = output / "reference.json" _json(request_path, reference_request) runner = ( - context.repository / "tools/benchmark_qualification/references/timm_image_classification.py" + context.repository / "qualification_tests/benchmark_qualification/references/timm_image_classification.py" ) completed = run_command( [ diff --git a/tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml b/qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml rename to qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml diff --git a/tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml b/qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml rename to qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml diff --git a/tools/benchmark_qualification/benchmarks/embedding_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/embedding_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/encoder_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/encoder_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/encoder_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/encoder_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml b/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml rename to qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml diff --git a/tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml b/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml diff --git a/tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml b/qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml diff --git a/tools/benchmark_qualification/benchmarks/image_classification_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/image_classification_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/image_classification_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/image_classification_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/image_features_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/image_features_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/imagenette_classification.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/imagenette_classification.yaml rename to qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml diff --git a/tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml b/qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml rename to qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml diff --git a/tools/benchmark_qualification/benchmarks/media_generation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/media_generation_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml b/qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml diff --git a/tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml b/qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml diff --git a/tools/benchmark_qualification/benchmarks/object_detection_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/object_detection_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/reranking_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/reranking_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/robot_action_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/robot_action_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/robot_action_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/robot_action_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/task_output_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/task_output_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml diff --git a/tools/benchmark_qualification/benchmarks/text_generation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/text_generation_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/time_series_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/time_series_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/vision_language_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/vision_language_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml b/qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml diff --git a/tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml b/qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml diff --git a/tools/benchmark_qualification/catalog.py b/qualification_tests/benchmark_qualification/catalog.py similarity index 98% rename from tools/benchmark_qualification/catalog.py rename to qualification_tests/benchmark_qualification/catalog.py index 17caf44f7b..548639dafa 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/qualification_tests/benchmark_qualification/catalog.py @@ -144,7 +144,7 @@ def select( def load_benchmark(repository: Path, case: QualificationCase) -> dict[str, Any]: - path = repository / "tools/benchmark_qualification/benchmarks" / f"{case.benchmark}.yaml" + path = repository / "qualification_tests/benchmark_qualification/benchmarks" / f"{case.benchmark}.yaml" value = _yaml_object(path, "benchmark definition") if value.get("kind") != case.kind: raise QualificationError( diff --git a/tools/benchmark_qualification/datasets.py b/qualification_tests/benchmark_qualification/datasets.py similarity index 100% rename from tools/benchmark_qualification/datasets.py rename to qualification_tests/benchmark_qualification/datasets.py diff --git a/qualification_tests/benchmark_qualification/performance/README.md b/qualification_tests/benchmark_qualification/performance/README.md new file mode 100644 index 0000000000..085c6fa31b --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/README.md @@ -0,0 +1,95 @@ + + +# Performance qualification + +This package is repository-only CI qualification. The installed user benchmark +remains `trtmc-bench` under `apps/benchmark`; it does not import this package. + +## Ownership + +- `matrix.py` selects and resumes multi-model campaigns. +- `runner.py` executes one resolved qualification case. +- `reference_protocol.py` defines the Accuracy/Performance reference command. +- `reference_harness.py` provides model-agnostic timing and result writing for + family-owned references. +- `reporting.py` produces `report.json` and `report.html`. +- `families//tests/benchmark/*.yaml` owns model cases, datasets, + metrics, gates, reference environment, and measurement settings. +- `families//tests/benchmark/reference.py` owns reference behavior that + cannot be expressed by a framework-generic adapter. + +Shared code must not select behavior from a family name. Adding a normal family +case changes only the family directory. Campaign configuration may select cases +or hardware, but does not redefine model behavior. + +## Execution + +The candidate runs through the installed `trtmc-bench` executable. The +reference runs in the Python environment prepared for its family, so conflicting +reference dependencies do not affect the candidate or another family. + +Run a checked-in campaign entry with: + +```bash +python3 -m qualification_tests.benchmark_qualification.performance check \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ + --entry gpt2.generate + +python3 -m qualification_tests.benchmark_qualification.performance run \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ + --entry gpt2.generate +``` + +Resume a run or regenerate its report with: + +```bash +python3 -m qualification_tests.benchmark_qualification.performance resume +python3 -m qualification_tests.benchmark_qualification.performance report +``` + +`tools/model_benchmark.py` is the thin per-model CI entry point. It discovers +family profiles and invokes the same single-case runner; it does not build a +temporary campaign or start a nested matrix process. + +## Reference contract + +A reference declares exactly one of: + +```yaml +reference: + adapter: hf-transformers-asr +``` + +or: + +```yaml +reference: + script: tests/benchmark/reference.py +``` + +Framework-generic adapters receive only the declared adapter and parameters. +Family scripts receive their own family identity but shared code never branches +on it. Both return `trtmc.perf-baseline/v1` with one finite positive timing +sample per measured iteration and the output summary required by the declared +contract. + +Reference model loading occurs before warmup. The three timing fields +`timing_scope`, `input_preparation_included`, and `asset_loading_included` must +match the profile. Model load, compilation, and warmup are excluded from timed +samples. + +## Results + +Each run retains raw commands, logs, artifacts, and `results.json`, then writes: + +- `report.json` for automation; +- `report.html` for inspection. + +An output-contract mismatch is a model `failed` result. Reference/candidate +command failures, invalid reports, missing artifacts, and other execution +problems are `error`, not model gate failures. diff --git a/qualification_tests/benchmark_qualification/performance/__init__.py b/qualification_tests/benchmark_qualification/performance/__init__.py new file mode 100644 index 0000000000..cbe4fa2c76 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Internal Performance qualification and campaign execution.""" + +from .matrix import main +from .runner import run_case + +__all__ = ["main", "run_case"] diff --git a/qualification_tests/benchmark_qualification/performance/__main__.py b/qualification_tests/benchmark_qualification/performance/__main__.py new file mode 100644 index 0000000000..b2aee43827 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/__main__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from .matrix import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/benchmark/performance/environments/auto-thor.yaml b/qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml similarity index 82% rename from apps/benchmark/performance/environments/auto-thor.yaml rename to qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml index 0b0d25f8c4..f14916b0a9 100644 --- a/apps/benchmark/performance/environments/auto-thor.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml @@ -7,8 +7,8 @@ name: auto-thor tools: trtmc_bench: apps/benchmark/trtmc-bench trtmc_worker: ${TRTMC_PERF_WORKER} - hf_transformers_runner: apps/benchmark/performance/baselines/hf_transformers.py - task_reference_runner: apps/benchmark/performance/baselines/task_reference.py + hf_transformers_runner: qualification_tests/benchmark_qualification/performance/references/hf_transformers.py + task_reference_runner: qualification_tests/benchmark_qualification/performance/references/generic_reference.py references: elf_repo: ${TRTMC_ELF_REFERENCE_REPO} diff --git a/apps/benchmark/performance/environments/gb300.yaml b/qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml similarity index 81% rename from apps/benchmark/performance/environments/gb300.yaml rename to qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml index 6eb03c0a66..6e991421b8 100644 --- a/apps/benchmark/performance/environments/gb300.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml @@ -7,8 +7,8 @@ name: gb300-ci tools: trtmc_bench: apps/benchmark/trtmc-bench trtmc_worker: ${TRTMC_PERF_WORKER} - hf_transformers_runner: apps/benchmark/performance/baselines/hf_transformers.py - task_reference_runner: apps/benchmark/performance/baselines/task_reference.py + hf_transformers_runner: qualification_tests/benchmark_qualification/performance/references/hf_transformers.py + task_reference_runner: qualification_tests/benchmark_qualification/performance/references/generic_reference.py references: elf_repo: ${TRTMC_ELF_REFERENCE_REPO} diff --git a/apps/benchmark/performance/environments/l4t-thor.yaml b/qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml similarity index 82% rename from apps/benchmark/performance/environments/l4t-thor.yaml rename to qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml index 8744f6191b..f536ff145e 100644 --- a/apps/benchmark/performance/environments/l4t-thor.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml @@ -7,8 +7,8 @@ name: l4t-thor tools: trtmc_bench: apps/benchmark/trtmc-bench trtmc_worker: ${TRTMC_PERF_WORKER} - hf_transformers_runner: apps/benchmark/performance/baselines/hf_transformers.py - task_reference_runner: apps/benchmark/performance/baselines/task_reference.py + hf_transformers_runner: qualification_tests/benchmark_qualification/performance/references/hf_transformers.py + task_reference_runner: qualification_tests/benchmark_qualification/performance/references/generic_reference.py references: elf_repo: ${TRTMC_ELF_REFERENCE_REPO} diff --git a/apps/benchmark/performance/release.yaml b/qualification_tests/benchmark_qualification/performance/config/release.yaml similarity index 97% rename from apps/benchmark/performance/release.yaml rename to qualification_tests/benchmark_qualification/performance/config/release.yaml index fcb7410073..f891fda4be 100644 --- a/apps/benchmark/performance/release.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/release.yaml @@ -147,8 +147,7 @@ entries: testcase: detr-resnet-50 baseline: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: object-detection} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -233,7 +232,7 @@ entries: testcase: canary-1b-v2 baseline: runner: task-reference - adapter: nemo-asr + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -291,8 +290,7 @@ entries: testcase: deepseek-ocr baseline: runner: task-reference - adapter: hf-transformers-vlm - adapter_options: {workflow: document-ocr-infer} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-pipeline-call-wall @@ -325,8 +323,7 @@ entries: testcase: dinov3-vits16-pretrain-lvd1689m baseline: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: image-features} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -415,7 +412,7 @@ entries: valid_iters: 8 baseline: runner: task-reference - adapter: upstream-fast-foundation-stereo + script: tests/benchmark/reference.py reference_inputs: model_dir: environment: fast_foundation_stereo_model @@ -559,7 +556,7 @@ entries: testcase: lance-3b-x2t-image baseline: runner: task-reference - adapter: upstream-lance + script: tests/benchmark/reference.py reference_inputs: reference_repo: {environment: lance_repo, required: [inference_lance.py]} mode: pytorch-eager @@ -579,7 +576,7 @@ entries: testcase: act-aloha-sim-transfer-cube-recorded-episode-0-frame-0 baseline: runner: task-reference - adapter: pytorch-lerobot-act + script: tests/benchmark/reference.py reference_inputs: source_root: {environment: lerobot_repo, required: [lerobot/common/policies/act/modeling_act.py]} mode: pytorch-eager @@ -607,8 +604,7 @@ entries: testcase: locateanything-3b baseline: runner: task-reference - adapter: hf-transformers-vlm - adapter_options: {workflow: grounded-generation} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -639,7 +635,7 @@ entries: max_new_tokens: 256 baseline: runner: task-reference - adapter: nemo-tts + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -747,8 +743,7 @@ entries: testcase: nemotron-3.5-asr-streaming-0.6b baseline: runner: task-reference - adapter: nemo-asr - adapter_options: {transcription_input: manifest, model_loader: prompted-nemo-archive} + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -821,7 +816,7 @@ entries: testcase: personaplex-7b baseline: runner: task-reference - adapter: pytorch-personaplex + script: tests/benchmark/reference.py reference_inputs: official_repo: {environment: personaplex_repo, required: [moshi]} mode: pytorch-eager @@ -924,7 +919,7 @@ entries: testcase: qwen3-omni-30b-a3b-instruct baseline: runner: task-reference - adapter: hf-qwen3-omni + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1000,8 +995,7 @@ entries: testcase: sam-vit-base baseline: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: point-prompted-segmentation} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1015,8 +1009,7 @@ entries: testcase: sam3 baseline: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: text-prompted-segmentation} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1031,7 +1024,7 @@ entries: testcase: sana-wm-bidirectional baseline: runner: task-reference - adapter: upstream-sana-wm + script: tests/benchmark/reference.py reference_inputs: reference_repo: {environment: sana_repo} model_dir: {environment: sana_model} @@ -1041,6 +1034,11 @@ entries: output_contract: media-shape adapter_options: intrinsics: assets/demo_0_intrinsics.npy + translation_speed: 0.055 + rotation_speed_deg: 1.2 + fps: 16 + flow_shift: 9.8 + no_action_overlay: true - id: segformer.segment family: segformer operation: segment @@ -1049,8 +1047,7 @@ entries: testcase: segformer-b0-ade baseline: runner: task-reference - adapter: hf-transformers-vision - adapter_options: {vision_task: semantic-segmentation} + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall diff --git a/apps/benchmark/performance/requirements.txt b/qualification_tests/benchmark_qualification/performance/config/requirements.txt similarity index 100% rename from apps/benchmark/performance/requirements.txt rename to qualification_tests/benchmark_qualification/performance/config/requirements.txt diff --git a/tools/perf_matrix.py b/qualification_tests/benchmark_qualification/performance/matrix.py similarity index 90% rename from tools/perf_matrix.py rename to qualification_tests/benchmark_qualification/performance/matrix.py index 89707c3771..7e25d1ed46 100755 --- a/tools/perf_matrix.py +++ b/qualification_tests/benchmark_qualification/performance/matrix.py @@ -7,7 +7,6 @@ from __future__ import annotations import argparse -import html import json import math import os @@ -21,7 +20,6 @@ import time from array import array from copy import deepcopy -from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from string import Template @@ -30,7 +28,7 @@ import yaml -REPOSITORY = Path(__file__).resolve().parents[1] +REPOSITORY = Path(__file__).resolve().parents[3] BUILDER_SOURCE = REPOSITORY / "core/builder" BENCHMARK_SOURCE = REPOSITORY / "apps/benchmark" MANIFEST_ROOT = REPOSITORY / "families" @@ -38,8 +36,10 @@ if str(source) not in sys.path: sys.path.insert(0, str(source)) -from apps.benchmark.performance.baselines.timing_contracts import timing_contract # noqa: E402 -from apps.benchmark.performance.baselines.hf_transformers import flatten_config # noqa: E402 +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract # noqa: E402 +from qualification_tests.benchmark_qualification.performance.references.hf_transformers import flatten_config # noqa: E402 +from qualification_tests.benchmark_qualification.performance import reference_protocol # noqa: E402 +from qualification_tests.benchmark_qualification.performance.reporting import write_report # noqa: E402 from trtmc_benchmark.artifact_metrics import ( # noqa: E402 metric_geometry_metrics, metric_geometry_passes, @@ -49,6 +49,16 @@ from trtmc_benchmark.catalog import ManifestCatalog, resolve_case, selected_task_for_case # noqa: E402 from trtmc_benchmark.task_adapters import default_operation # noqa: E402 from trtmc_benchmark.types import BenchmarkError # noqa: E402 +from qualification_tests.benchmark_qualification.performance.stability import ( # noqa: E402 + measurement_stability as _measurement_stability, + p50 as _p50, + timing_stability as _timing_stability, # noqa: F401 - compatibility for existing tests +) +from qualification_tests.benchmark_qualification.performance.types import ( # noqa: E402 + Environment, + PerfMatrixError, + ResolvedEntry, +) SUITE_SCHEMA = "trtmc.perf-suite/v2" @@ -101,42 +111,6 @@ "transcription-text", "vision-language-text", } -class PerfMatrixError(RuntimeError): - pass - - -@dataclass(frozen=True) -class Environment: - name: str - trtmc_bench: Path - worker: Path - hf_runner: Path - task_runner: Path - results_root: Path - scratch_root: Path - bundle_cache: Path - bundle_roots: tuple[Path, ...] - runtime_root: Path - bundle_retention: str - local_files_only: bool - timeout_seconds: int - references: Mapping[str, str] - reference_python: Path = Path(sys.executable) - storage_root: Path | None = None - hf_cache_mode: str = "shared" - hf_cache_retention: str = "retain" - - -@dataclass(frozen=True) -class ResolvedEntry: - spec: Mapping[str, Any] - model: Any - case: Any - manifest: Mapping[str, Any] - reference_precision: str - baseline_timing: Mapping[str, Any] - - def parser() -> argparse.ArgumentParser: value = argparse.ArgumentParser(description=__doc__) commands = value.add_subparsers(dest="command", required=True) @@ -197,7 +171,7 @@ def load_suite(path: Path) -> tuple[str, list[dict[str, Any]], set[str]]: if owner is not None: _family_file(owner, "tests/performance.yaml", "performance suite") name, entries, excluded = _load_suite_file(path, owner=owner) - canonical = REPOSITORY / "apps/benchmark/performance/release.yaml" + canonical = REPOSITORY / "qualification_tests/benchmark_qualification/performance/config/release.yaml" if path.resolve() != canonical.resolve(): return name, entries, excluded positions = {entry["id"]: index for index, entry in enumerate(entries)} @@ -885,13 +859,15 @@ def _baseline_command( source_placement = baseline.get("source_language_placement") if source_placement is not None: request_value["source_language_placement"] = source_placement - request = json.dumps(request_value, ensure_ascii=True, separators=(",", ":")) model = _baseline_model(entry, environment) + revision = _baseline_revision(entry, model) + local_files_only = bool(baseline.get("local_files_only", environment.local_files_only)) + trust_remote_code = bool(entry.manifest.get("trust_remote_code", False)) common = [ "--model", model, "--request-json", - request, + json.dumps(request_value, ensure_ascii=True, separators=(",", ":")), "--precision", entry.reference_precision, "--mode", @@ -932,42 +908,40 @@ def _baseline_command( if bool(baseline.get("dynamic", True)): arguments.append("--compile-dynamic") else: - executable = ( - [str(_family_script(entry.spec))] - if "script" in baseline - else [str(environment.task_runner), "--adapter", str(baseline["adapter"])] - ) - arguments = [ - str(getattr(environment, "reference_python", sys.executable)), - *executable, - "--family", - entry.model.family, - "--operation", - str(entry.spec["operation"]), - "--manifest", - str(entry.model.manifest_path), - "--adapter-options-json", - json.dumps( - _adapter_options(entry, environment), - ensure_ascii=True, - separators=(",", ":"), + return reference_protocol.command( + python=getattr(environment, "reference_python", Path(sys.executable)), + generic_runner=environment.task_runner, + script=_family_script(entry.spec) if "script" in baseline else None, + family=entry.model.family, + operation=str(entry.spec["operation"]), + manifest=entry.model.manifest_path, + selected_task=( + _effective_task(entry) + if "script" in baseline or entry.case.selected_task is not None + else None ), - "--timing-contract-json", - json.dumps(entry.baseline_timing, ensure_ascii=True, separators=(",", ":")), - "--padding", - str(baseline.get("padding", "longest")), - *common, - ] - if "script" not in baseline: - arguments.extend(("--testcase-name", entry.case.testcase_name)) - if "script" in baseline or entry.case.selected_task is not None: - arguments.extend(("--selected-task", _effective_task(entry))) - revision = _baseline_revision(entry, model) + testcase_name=entry.case.testcase_name, + adapter=None if "script" in baseline else str(baseline["adapter"]), + adapter_options=_adapter_options(entry, environment), + timing_contract=entry.baseline_timing, + padding=str(baseline.get("padding", "longest")), + model=model, + revision=revision, + request=request_value, + precision=entry.reference_precision, + mode=selected_mode, + warmup=entry.case.measurement.warmup, + iterations=entry.case.measurement.iterations, + case_name=str(entry.spec["id"]), + output=output, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) if revision: arguments.extend(("--revision", revision)) - if bool(entry.manifest.get("trust_remote_code", False)): + if trust_remote_code: arguments.append("--trust-remote-code") - if bool(baseline.get("local_files_only", environment.local_files_only)): + if local_files_only: arguments.append("--local-files-only") return arguments @@ -1123,94 +1097,6 @@ def _candidate_result(path: Path) -> dict[str, Any]: } -def _p50(value: Mapping[str, Any]) -> float: - metrics = value.get("metrics", {}) - latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} - p50 = latency.get("p50") if isinstance(latency, Mapping) else None - if isinstance(p50, bool) or not isinstance(p50, (int, float)) or not math.isfinite(float(p50)): - raise PerfMatrixError("measurement has no finite latency p50") - return float(p50) - - -_TIMING_STABILITY_SAMPLE_COUNT = 10 -_TIMING_STABILITY_MAX_HALF_CHANGE_PERCENT = 5.0 -_TIMING_STABILITY_MEDIAN_BAND_PERCENT = 5.0 -_TIMING_STABILITY_MIN_IN_BAND = 8 - - -def _timing_stability(values: Sequence[Any]) -> dict[str, Any]: - if len(values) != _TIMING_STABILITY_SAMPLE_COUNT: - return { - "status": "not_evaluated", - "sample_count": len(values), - "reason": "requires_10_samples", - } - try: - samples = [float(value) for value in values] - except (TypeError, ValueError): - return { - "status": "not_evaluated", - "sample_count": len(values), - "reason": "invalid_samples", - } - if not all(math.isfinite(value) and value > 0.0 for value in samples): - return { - "status": "not_evaluated", - "sample_count": len(samples), - "reason": "invalid_samples", - } - - middle = len(samples) // 2 - median_ms = float(statistics.median(samples)) - first_half_median_ms = float(statistics.median(samples[:middle])) - second_half_median_ms = float(statistics.median(samples[middle:])) - half_change_percent = ( - abs(second_half_median_ms - first_half_median_ms) / first_half_median_ms * 100.0 - ) - samples_within_band = sum( - abs(sample - median_ms) / median_ms * 100.0 <= _TIMING_STABILITY_MEDIAN_BAND_PERCENT - for sample in samples - ) - stable = ( - half_change_percent <= _TIMING_STABILITY_MAX_HALF_CHANGE_PERCENT - and samples_within_band >= _TIMING_STABILITY_MIN_IN_BAND - ) - return { - "status": "stable" if stable else "unstable", - "sample_count": len(samples), - "median_ms": median_ms, - "first_half_median_ms": first_half_median_ms, - "second_half_median_ms": second_half_median_ms, - "half_median_change_percent": half_change_percent, - "samples_within_band": samples_within_band, - } - - -def _measurement_stability( - reference: Mapping[str, Any], candidate: Mapping[str, Any] -) -> dict[str, Any]: - reference_result = _timing_stability(reference.get("samples_ms", [])) - candidate_result = _timing_stability(candidate.get("samples_ms", [])) - statuses = {reference_result["status"], candidate_result["status"]} - if statuses == {"stable"}: - status = "stable" - elif "unstable" in statuses: - status = "unstable" - else: - status = "not_evaluated" - return { - "status": status, - "policy": { - "required_samples": _TIMING_STABILITY_SAMPLE_COUNT, - "max_half_median_change_percent": _TIMING_STABILITY_MAX_HALF_CHANGE_PERCENT, - "median_band_percent": _TIMING_STABILITY_MEDIAN_BAND_PERCENT, - "minimum_samples_within_band": _TIMING_STABILITY_MIN_IN_BAND, - }, - "reference": reference_result, - "candidate": candidate_result, - } - - def compare( entry: ResolvedEntry, candidate: Mapping[str, Any], @@ -2406,7 +2292,7 @@ def _entry_slug(entry_id: str) -> str: return "entry" if value in {"", ".", ".."} else value -def _execute_entry( +def execute_entry( entry: ResolvedEntry, environment: Environment, run_directory: Path, @@ -2707,7 +2593,7 @@ def _run_rows( while (artifact_root / f"attempt-{attempt}").exists(): attempt += 1 try: - row = _execute_entry( + row = execute_entry( entry, environment, run_directory, @@ -2744,180 +2630,6 @@ def _run_rows( return 0 if results["status"] == "completed" else 1 -def write_report( - run_directory: Path, - results: Mapping[str, Any], - preparation: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - source_schema = results.get("schema_version", RESULT_SCHEMA) - row_key = "cases" if source_schema == "trtmc.perf-matrix/v1" else "rows" - rows = [dict(row) for row in results.get(row_key, []) if isinstance(row, Mapping)] - selected_ids = results.get("selected_entry_ids") - if not isinstance(selected_ids, list) or not all( - isinstance(value, str) for value in selected_ids - ): - raise PerfMatrixError("matrix results selected_entry_ids must be strings") - completed_ids = {str(row.get("id")) for row in rows} - counts = { - status: sum(row.get("status") == status for row in rows) - for status in ("green", "yellow", "red", "contract-mismatch", "white") - } - report = { - "schema_version": REPORT_SCHEMA, - "source_schema_version": source_schema, - "generated_at": _now(), - "status": results.get("status", "unknown"), - "suite": results.get("suite"), - "environment": results.get("environment"), - "summary": { - "selected": len(selected_ids), - "pending": sum(entry_id not in completed_ids for entry_id in selected_ids), - "comparable": counts["green"] + counts["yellow"] + counts["red"], - **counts, - }, - "rows": rows, - } - if preparation is not None: - report["preparation"] = dict(preparation) - _write_json(run_directory / "report.json", report) - (run_directory / "report.html").write_text(_report_html(report), encoding="utf-8") - return report - - -def _measurement_html(value: Any) -> str: - if not isinstance(value, Mapping): - return "Not recorded" - metrics = value.get("metrics", {}) - latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} - samples = value.get("samples_ms", []) - measured = ( - sorted( - float(sample) - for sample in samples - if isinstance(sample, (int, float)) - and not isinstance(sample, bool) - and math.isfinite(sample) - and sample > 0 - ) - if isinstance(samples, list) - else [] - ) - parts = [] - for name, percentile in (("p50", 0.5), ("p95", 0.95)): - number = latency.get(name) if isinstance(latency, Mapping) else None - if number is None and measured: - position = (len(measured) - 1) * percentile - lower, upper = math.floor(position), math.ceil(position) - number = measured[lower] + (measured[upper] - measured[lower]) * (position - lower) - if ( - isinstance(number, (int, float)) - and not isinstance(number, bool) - and math.isfinite(number) - ): - parts.append(f"{name}: {number:,.3f} ms") - policy = value.get("measurement_policy", {}) - scope = value.get("timing_scope") or ( - policy.get("timing_scope") if isinstance(policy, Mapping) else None - ) - parts.append("Scope: " + html.escape(str(scope or "not recorded"))) - if isinstance(metrics, Mapping): - parts.extend( - f"{html.escape(name)}: {number:,.3f}" - for name, number in metrics.items() - if name.endswith("_per_s") - and isinstance(number, (int, float)) - and not isinstance(number, bool) - and math.isfinite(number) - ) - return "
".join(parts) - - -def _report_html(report: Mapping[str, Any]) -> str: - rows = [] - for row in report.get("rows", []): - comparison = row.get("comparison", {}) if isinstance(row, Mapping) else {} - measurement_stability = ( - row.get("measurement_stability", {}) if isinstance(row, Mapping) else {} - ) - candidate = _measurement_html( - row.get("candidate") - or {"metrics": {"latency_ms": {"p50": comparison.get("candidate_p50_ms")}}} - ) - reference = _measurement_html( - row.get("reference", row.get("baseline")) - or {"metrics": {"latency_ms": {"p50": comparison.get("reference_p50_ms")}}} - ) - reason = comparison.get("reason", row.get("error", "")) - stability = ( - measurement_stability.get("status", "") - if isinstance(measurement_stability, Mapping) - else "" - ) - evidence = { - key: row[key] - for key in ( - "commands", - "comparison", - "measurement_stability", - "resolved_settings", - "baseline_contract", - ) - if key in row - } - detail = html.escape(json.dumps(evidence, indent=2, sort_keys=True)) - link = "" - artifact = row.get("artifact_dir") - if ( - isinstance(artifact, str) - and artifact - and not Path(artifact).is_absolute() - and ".." not in Path(artifact).parts - and not any(value in artifact for value in (":", "\\")) - ): - from urllib.parse import quote - - link = f'Case artifacts' - rows.append( - "" - f"{html.escape(str(row.get('id', '')))}" - f"{html.escape(str(row.get('model', '')))}" - f"{html.escape(str(row.get('operation', '')))}" - f"{html.escape(str(row.get('status', '')))}" - f"{html.escape(str(stability))}" - f"{candidate}" - f"{reference}" - f"{html.escape(str(reason))}" - f"

Evidence / commands{link}
{detail}
" - "" - ) - return f""" -TRTMC performance - -

TRTMC performance matrix

-

Source data: {html.escape(str(report.get("source_schema_version", RESULT_SCHEMA)))}. Original timing scopes and recorded outcomes are preserved.

-

Status: {html.escape(str(report.get("status", "unknown")))}; -selected: {html.escape(str(report.get("summary", {}).get("selected", 0)))}; -pending: {html.escape(str(report.get("summary", {}).get("pending", 0)))}

- - - -{"".join(rows)}
EntryModelOperationStatusStabilityCandidate measurementsReference measurementsReasonEvidence
-

Green is faster by more than the margin, yellow is within the margin, red is -slower by more than the margin, and white is not comparable.

-

Output comparison follows the suite's declared contract; matching dimensions or token counts alone does not establish model correctness. Use family correctness tests for acceptance.

-

Original results · Report data

- - -""" - def prepare_entries( entries: Sequence[ResolvedEntry], diff --git a/tools/benchmark_qualification/performance.py b/qualification_tests/benchmark_qualification/performance/qualification.py similarity index 62% rename from tools/benchmark_qualification/performance.py rename to qualification_tests/benchmark_qualification/performance/qualification.py index 80e4ebac9d..c886701616 100644 --- a/tools/benchmark_qualification/performance.py +++ b/qualification_tests/benchmark_qualification/performance/qualification.py @@ -8,19 +8,17 @@ import json import os from pathlib import Path -import sys from typing import Any, Mapping -import yaml +from qualification_tests.benchmark_qualification.performance import matrix as performance_matrix +from qualification_tests.benchmark_qualification.performance.runner import run_case -from .catalog import QualificationCase, QualificationError, load_benchmark -from .runtime import ( +from ..catalog import QualificationCase, QualificationError, load_benchmark +from ..runtime import ( RuntimeContext, - benchmark_executable, reference_environment_options, reference_python, require_candidate, - run_command, write_model_descriptor, write_result, ) @@ -91,30 +89,24 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st baseline = _resolve_reference_assets(case, context, baseline) descriptor = write_model_descriptor(case, output, request, context=context) entry_id = f"qualification.{case.family}.{case.name}" - suite = { - "schema_version": "trtmc.perf-suite/v2", - "name": case.id, - "entries": [ - { - "id": entry_id, - "family": case.family, - "operation": str(configured["operation"]), - "model": case.model, - "manifest": str(descriptor), - "workload": { - "testcase": case.name, - "request": dict(request), - }, - "measurement": { - "warmup": int(measurement.get("warmup", 5)), - "iterations": int(measurement.get("iterations", 10)), - }, - "baseline": {**dict(baseline), **dict(reference_timing)}, - "equivalence_margin_percent": float( - configured.get("equivalence_margin_percent", 5.0) - ), - } - ], + spec = { + "id": entry_id, + "family": case.family, + "operation": str(configured["operation"]), + "model": case.model, + "manifest": str(descriptor), + "workload": { + "testcase": case.name, + "request": dict(request), + }, + "measurement": { + "warmup": int(measurement.get("warmup", 5)), + "iterations": int(measurement.get("iterations", 10)), + }, + "baseline": {**dict(baseline), **dict(reference_timing)}, + "equivalence_margin_percent": float( + configured.get("equivalence_margin_percent", 5.0) + ), } if stability != { "samples": 10, @@ -124,82 +116,42 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st "retries": 1, }: raise QualificationError("unsupported Performance stability definition") - suite_path = output / "resolved-suite.yaml" - suite_path.write_text(yaml.safe_dump(suite, sort_keys=False), encoding="utf-8") results_root = output / "matrix" - environment = { - "schema_version": "trtmc.perf-environment/v2", - "name": "qualification", - "tools": { - "trtmc_bench": str(benchmark_executable(case, context)), - "trtmc_worker": str(worker), - "hf_transformers_runner": str( - context.repository / "apps/benchmark/performance/baselines/hf_transformers.py" - ), - "task_reference_runner": str( - context.repository / "apps/benchmark/performance/baselines/task_reference.py" - ), - "reference_python": str(reference_python(case, context)), - }, - "references": {}, - "storage": { - "results_root": str(results_root), - "scratch_root": str(output / "scratch"), - "bundle_cache": str(context.bundle_cache), - "bundle_roots": [str(path) for path in context.bundle_roots], - "runtime_root": str(runtime_root), - "bundle_retention": "retain", - }, - "execution": { - "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", - "hf_cache_mode": "shared", - "hf_cache_retention": "retain", - "timeout_seconds": int(configured.get("timeout_seconds", 7200)), - }, - } - environment_path = output / "resolved-environment.yaml" - environment_path.write_text(yaml.safe_dump(environment, sort_keys=False), encoding="utf-8") - command = [ - sys.executable, - str(context.repository / "tools/perf_matrix.py"), - "run", - str(suite_path), - "--environment", - str(environment_path), - "--entry", - entry_id, - "--allow-partial", - ] - if context.no_build: - command.append("--no-build") - if context.verbose: - command.append("--verbose") - completed = run_command( - command, - output, - "performance", - timeout=int(configured.get("timeout_seconds", 7200)) * 2, - verbose=context.verbose, + environment = performance_matrix.Environment( + name="qualification", + trtmc_bench=context.trtmc_bench, + worker=worker, + hf_runner=context.repository / "qualification_tests/benchmark_qualification/performance/references/hf_transformers.py", + task_runner=context.repository / "qualification_tests/benchmark_qualification/performance/references/generic_reference.py", + reference_python=reference_python(case, context), + results_root=results_root, + scratch_root=output / "scratch", + bundle_cache=context.bundle_cache, + bundle_roots=context.bundle_roots, + runtime_root=runtime_root, + bundle_retention="retain", + local_files_only=os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + timeout_seconds=int(configured.get("timeout_seconds", 7200)), + references={}, + hf_cache_mode="shared", + hf_cache_retention="retain", ) - run_directories = sorted(path.parent for path in results_root.glob("*/results.json")) - if not run_directories: - raise QualificationError(f"Performance produced no result; see {output}") - run_directory = run_directories[-1] try: - matrix = json.loads((run_directory / "results.json").read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - raise QualificationError( - f"Performance produced an unreadable result; see {run_directory}" - ) from error - if not isinstance(matrix, Mapping): - raise QualificationError(f"Performance produced an invalid result; see {run_directory}") - rows = matrix.get("rows") - row = ( - rows[0] - if isinstance(rows, list) and len(rows) == 1 and isinstance(rows[0], Mapping) - else {} + row = run_case( + spec, + environment, + results_root, + no_build=context.no_build, + verbose=context.verbose, + ) + except (OSError, performance_matrix.PerfMatrixError) as error: + raise QualificationError(f"Performance execution failed: {error}") from error + matrix_status = "completed" if row.get("status") in _COMPLETED_COMPARISONS else "failed" + status, error = _qualification_status( + 0 if matrix_status == "completed" else 1, + {"status": matrix_status}, + row, ) - status, error = _qualification_status(completed.returncode, matrix, row) comparison_value = row.get("comparison") comparison = comparison_value if isinstance(comparison_value, Mapping) else {} result = { @@ -209,7 +161,7 @@ def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[st "status": status, "model": case.model, "benchmark": case.benchmark, - "matrix_run": str(run_directory), + "matrix_run": str(results_root), "comparison_status": row.get("status") if isinstance(row, Mapping) else None, "metrics": { "candidate_p50_ms": comparison.get("candidate_p50_ms"), diff --git a/qualification_tests/benchmark_qualification/performance/reference_harness.py b/qualification_tests/benchmark_qualification/performance/reference_harness.py new file mode 100644 index 0000000000..48d0012faf --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/reference_harness.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic execution and reporting for family-owned Performance references.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import math +from pathlib import Path +import platform +import statistics +import time +from typing import Any, Callable, Mapping, Sequence + +from .references.hf_transformers import flatten_config +from .references.timing_contracts import timing_contract + + +@dataclass(frozen=True) +class Session: + """A loaded reference and the repeatable operation to measure.""" + + invoke: Callable[[], Mapping[str, Any]] + backend: str + timing_scope: str = "task-model-call-wall" + input_preparation_included: bool = False + asset_loading_included: bool = False + materialize: Callable[[dict[str, Any], Path], None] | None = None + + +@dataclass(frozen=True) +class Measurement: + """Already measured family output using the shared result contract.""" + + samples_ms: Sequence[float] + output_summary: Mapping[str, Any] + backend: str + timing_scope: str = "task-model-call-wall" + input_preparation_included: bool = False + asset_loading_included: bool = False + + +def parser(description: str) -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=description) + value.add_argument("--family", required=True) + value.add_argument("--operation", required=True) + value.add_argument("--manifest", required=True, type=Path) + value.add_argument("--adapter-options-json", default="{}") + value.add_argument("--timing-contract-json", required=True) + value.add_argument("--padding", default="longest") + value.add_argument("--model", required=True) + value.add_argument("--revision") + value.add_argument("--request-json", required=True) + value.add_argument("--precision", required=True, choices=("fp16", "bf16", "fp32")) + value.add_argument("--mode", required=True) + value.add_argument("--warmup", required=True, type=int) + value.add_argument("--iterations", required=True, type=int) + value.add_argument("--case-name", required=True) + value.add_argument("--testcase-name") + value.add_argument("--output", required=True, type=Path) + value.add_argument("--selected-task", required=True) + value.add_argument("--trust-remote-code", action="store_true") + value.add_argument("--local-files-only", action="store_true") + return value + + +def json_object(raw: str, label: str) -> dict[str, Any]: + try: + value = json.loads(raw) + except json.JSONDecodeError as error: + raise ValueError(f"{label} is not valid JSON: {error}") from error + if not isinstance(value, dict): + raise ValueError(f"{label} must be an object") + return value + + +def synchronize() -> None: + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.synchronize() + except ImportError: + pass + + +def measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: + if warmup < 0 or iterations < 1: + raise ValueError("warmup must be non-negative and iterations must be positive") + output: Mapping[str, Any] = {} + for _ in range(warmup): + output = session.invoke() + synchronize() + samples = [] + for _ in range(iterations): + synchronize() + started = time.perf_counter() + output = session.invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + if not all(math.isfinite(value) and value > 0.0 for value in samples): + raise RuntimeError("reference produced an invalid timing sample") + return samples, dict(output) + + +def environment() -> dict[str, Any]: + value: dict[str, Any] = { + "platform": platform.platform(), + "python": platform.python_version(), + } + try: + import torch + + value["torch"] = str(torch.__version__) + if torch.cuda.is_available(): + value["gpu"] = torch.cuda.get_device_name(0) + value["cuda"] = torch.version.cuda + except ImportError: + pass + return value + + +def run( + argv: Sequence[str], + *, + description: str, + load: Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session], +) -> int: + arguments = parser(description).parse_args(argv) + request = flatten_config(json_object(arguments.request_json, "--request-json")) + options = json_object(arguments.adapter_options_json, "--adapter-options-json") + load_started = time.perf_counter() + session = load(arguments, request, options) + load_seconds = time.perf_counter() - load_started + samples, output_summary = measure(session, arguments.warmup, arguments.iterations) + if session.materialize is not None: + session.materialize(output_summary, arguments.output) + return _write( + arguments, + Measurement( + samples, + output_summary, + session.backend, + session.timing_scope, + session.input_preparation_included, + session.asset_loading_included, + ), + load_seconds=load_seconds, + ) + + +def run_premeasured( + argv: Sequence[str], + *, + description: str, + execute: Callable[ + [argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Measurement + ], +) -> int: + arguments = parser(description).parse_args(argv) + request = flatten_config(json_object(arguments.request_json, "--request-json")) + options = json_object(arguments.adapter_options_json, "--adapter-options-json") + measurement = execute(arguments, request, options) + return _write(arguments, measurement, load_seconds=None) + + +def _write( + arguments: argparse.Namespace, + measurement: Measurement, + *, + load_seconds: float | None, +) -> int: + samples = [float(value) for value in measurement.samples_ms] + if len(samples) != arguments.iterations or not all( + math.isfinite(value) and value > 0.0 for value in samples + ): + raise RuntimeError("reference must return one finite positive sample per iteration") + actual = { + "timing_scope": measurement.timing_scope, + "input_preparation_included": measurement.input_preparation_included, + "asset_loading_included": measurement.asset_loading_included, + } + declared = json_object(arguments.timing_contract_json, "--timing-contract-json") + expected = timing_contract(runner="task-reference", declared=declared) if declared else None + if expected is not None and any(actual[name] != expected[name] for name in actual): + raise RuntimeError(f"family reference timing drifted: actual={actual}, declared={expected}") + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": measurement.backend, + "framework": measurement.backend, + "family": arguments.family, + "operation": arguments.operation, + "selected_task": arguments.selected_task, + "mode": arguments.mode, + "precision": arguments.precision, + "padding": arguments.padding, + "model": arguments.model, + "case_name": arguments.case_name, + "model_load_included": False, + "model_load_seconds": load_seconds, + **actual, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": { + **actual, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": statistics.median(samples), + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": dict(measurement.output_summary), + "environment": environment(), + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 diff --git a/qualification_tests/benchmark_qualification/performance/reference_protocol.py b/qualification_tests/benchmark_qualification/performance/reference_protocol.py new file mode 100644 index 0000000000..e2b7db98fd --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/reference_protocol.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command protocol shared by Accuracy and Performance reference execution.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + + +def command( + *, + python: Path, + generic_runner: Path, + script: Path | None, + family: str, + operation: str, + manifest: Path, + selected_task: str | None, + testcase_name: str | None, + adapter: str | None, + adapter_options: Mapping[str, Any], + timing_contract: Mapping[str, Any], + padding: str, + model: str, + revision: str | None, + request: Mapping[str, Any], + precision: str, + mode: str, + warmup: int, + iterations: int, + case_name: str, + output: Path, + trust_remote_code: bool, + local_files_only: bool, +) -> list[str]: + if (script is None) == (adapter is None): + raise ValueError("reference requires exactly one of script or adapter") + arguments = [ + str(python), + str(script or generic_runner), + ] + if script is not None: + arguments.extend(("--family", family)) + else: + arguments.extend(("--adapter", str(adapter))) + arguments.extend( + ( + "--operation", + operation, + "--manifest", + str(manifest), + "--adapter-options-json", + json.dumps(dict(adapter_options), ensure_ascii=True, separators=(",", ":")), + "--timing-contract-json", + json.dumps(dict(timing_contract), ensure_ascii=True, separators=(",", ":")), + "--padding", + padding, + "--model", + model, + "--request-json", + json.dumps(dict(request), ensure_ascii=True, separators=(",", ":")), + "--precision", + precision, + "--mode", + mode, + "--warmup", + str(warmup), + "--iterations", + str(iterations), + "--case-name", + case_name, + "--output", + str(output), + ) + ) + if script is None and testcase_name is not None: + arguments.extend(("--testcase-name", testcase_name)) + if selected_task is not None: + arguments.extend(("--selected-task", selected_task)) + if revision: + arguments.extend(("--revision", revision)) + if trust_remote_code: + arguments.append("--trust-remote-code") + if local_files_only: + arguments.append("--local-files-only") + return arguments diff --git a/qualification_tests/benchmark_qualification/performance/references/__init__.py b/qualification_tests/benchmark_qualification/performance/references/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/benchmark/performance/baselines/audio_reference.py b/qualification_tests/benchmark_qualification/performance/references/audio_reference.py similarity index 100% rename from apps/benchmark/performance/baselines/audio_reference.py rename to qualification_tests/benchmark_qualification/performance/references/audio_reference.py diff --git a/qualification_tests/benchmark_qualification/performance/references/generic_reference.py b/qualification_tests/benchmark_qualification/performance/references/generic_reference.py new file mode 100644 index 0000000000..43f75c0015 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/references/generic_reference.py @@ -0,0 +1,1685 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmark task-aligned non-text model reference operations.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import math +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Callable, Mapping, Sequence + + +REPOSITORY = Path(__file__).resolve().parents[4] +for source_root in ( + REPOSITORY, + REPOSITORY / "core/builder", + REPOSITORY / "apps/benchmark", +): + if str(source_root) not in sys.path: + sys.path.insert(0, str(source_root)) + +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract # noqa: E402 +from qualification_tests.benchmark_qualification.performance.references.hf_transformers import _batch_prompt, flatten_config # noqa: E402 + +ADAPTERS = ( + "hf-diffusers", + "hf-diffusers-modular", + "hf-transformers-asr", + "hf-transformers-embedding", + "hf-transformers-reranking", + "hf-transformers-tts", + "hf-transformers-vision", + "hf-transformers-vlm", + "pytorch-timeseries", + "timm-classification", +) +PYTORCH_ADAPTERS = {"pytorch-timeseries"} + + +@dataclass(frozen=True) +class Session: + """One loaded reference model and its repeatable timed operation.""" + + invoke: Callable[[], Any] + framework: str + timing_scope: str = "task-model-call-wall" + input_preparation_included: bool = False + asset_loading_included: bool = False + compile_evidence: dict[str, Any] | None = None + summarize: Callable[[Any], Mapping[str, Any]] | None = None + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adapter", required=True, choices=ADAPTERS) + parser.add_argument("--operation", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--revision") + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument( + "--selected-task", help="Semantic Task selected independently of bundle identity" + ) + parser.add_argument("--request-json", required=True) + parser.add_argument("--adapter-options-json", default="{}") + parser.add_argument("--timing-contract-json", default="{}") + parser.add_argument("--precision", required=True, choices=("fp16", "fp32", "bf16")) + parser.add_argument( + "--mode", required=True, choices=("hf-eager", "pytorch-eager", "torch-compile") + ) + parser.add_argument("--padding", default="longest") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument( + "--testcase-name", help="Selected manifest testcase, not the performance entry ID" + ) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{label} is not valid JSON: {exc}") from exc + if not isinstance(value, dict): + raise ValueError(f"{label} must contain an object") + return value + + +def _selected_task(arguments: argparse.Namespace) -> str: + selected = getattr(arguments, "selected_task", None) + if selected is None: + return json.loads(arguments.manifest.read_text(encoding="utf-8"))["task"] + if not isinstance(selected, str) or not selected or selected != selected.strip(): + raise ValueError("selected_task must be a nonempty Task ID without surrounding whitespace") + return selected + + +def _torch_dtype(torch_module: Any, precision: str) -> Any: + return { + "fp16": torch_module.float16, + "fp32": torch_module.float32, + "bf16": torch_module.bfloat16, + }[precision] + + +def _seed_all(torch_module: Any, seed: int) -> None: + import random + + import numpy as np + + random.seed(seed) + np.random.seed(seed) + torch_module.manual_seed(seed) + if torch_module.cuda.is_available(): + torch_module.cuda.manual_seed_all(seed) + + +def _request_seed(request: Mapping[str, Any], default: Any = 42) -> int: + value = request.get("seed", default) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("reference request seed must be an integer") + return value + + +def _bark_generation_options(request: Mapping[str, Any]) -> dict[str, int]: + max_new_tokens = request.get("max_new_tokens", 0) + if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): + raise ValueError("Bark request max_new_tokens must be an integer") + if max_new_tokens <= 0: + return {} + return {"semantic_max_new_tokens": max_new_tokens} + + +def _load_kwargs(arguments: argparse.Namespace, torch_module: Any) -> dict[str, Any]: + values: dict[str, Any] = { + "trust_remote_code": arguments.trust_remote_code, + "local_files_only": arguments.local_files_only, + "torch_dtype": _torch_dtype(torch_module, arguments.precision), + } + if arguments.revision: + values["revision"] = arguments.revision + return values + + +def _processor_kwargs(arguments: argparse.Namespace) -> dict[str, Any]: + values: dict[str, Any] = { + "trust_remote_code": arguments.trust_remote_code, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + values["revision"] = arguments.revision + return values + + +def _cached_snapshot_path(repo_id: str, requested: str | None, marker_file: str) -> Path | None: + if Path(repo_id).exists(): + return Path(repo_id).resolve() + try: + from huggingface_hub import try_to_load_from_cache + + cached = try_to_load_from_cache( + repo_id=repo_id, + filename=marker_file, + revision=requested or "main", + ) + except (ImportError, OSError, ValueError): + return None + if isinstance(cached, str) and Path(cached).is_file(): + return Path(cached).parent.resolve() + return None + + +def _to_device(value: Any, device: Any, dtype: Any = None) -> Any: + if isinstance(value, Mapping): + return {name: _to_device(item, device, dtype) for name, item in value.items()} + if hasattr(value, "to"): + if dtype is not None and getattr(value, "is_floating_point", lambda: False)(): + return value.to(device=device, dtype=dtype) + return value.to(device) + return value + + +def _asset_path(arguments: argparse.Namespace, request: Mapping[str, Any], key: str) -> Path: + raw = str(request.get(key, "") or "") + if not raw: + raise ValueError(f"{arguments.adapter} reference requires request.{key}") + path = Path(raw) + if not path.is_absolute(): + path = arguments.manifest.resolve().parent.parent / path + if not path.is_file(): + raise FileNotFoundError(f"reference input does not exist: {path}") + return path + + +def _tensor_summary(value: Any) -> dict[str, Any]: + shape = [int(dim) for dim in value.shape] + return { + "shape": shape, + "element_count": int(value.numel()), + "finite": bool(value.isfinite().all().item()), + } + + +def _flatten_tensor_values(value: Any) -> list[float]: + flattened: list[float] = [] + + def visit(item: Any) -> None: + if isinstance(item, (list, tuple)): + for child in item: + visit(child) + else: + flattened.append(float(item)) + + visit(value.detach().float().cpu().tolist()) + return flattened + + +def _forecast_summary( + value: Any, task: str, quantile_levels: Sequence[float] = () +) -> dict[str, Any]: + summary = _tensor_summary(value) + summary["values"] = _flatten_tensor_values(value) + if task not in {"series_to_point_forecast", "series_to_quantile_forecast"}: + return summary + shape = summary["shape"] + if len(shape) not in {2, 3} or shape[0] != 1: + raise ValueError("forecast reference requires an explicit one-series batch axis") + if task == "series_to_point_forecast": + horizon, channels = shape[1], shape[2] if len(shape) == 3 else 1 + summary.update(shape=[horizon, channels], axes=["horizon", "channel"]) + else: + if len(shape) != 3 or len(quantile_levels) != shape[1]: + raise ValueError( + "quantile forecast reference requires exact checkpoint quantile levels" + ) + horizon = shape[2] + summary.update( + shape=[shape[1], horizon, 1], + axes=["quantile", "horizon", "channel"], + quantile_levels=list(quantile_levels), + ) + summary.update( + forecast_elements=summary["element_count"], horizon_steps=list(range(1, horizon + 1)) + ) + return summary + + +def _regression_values_summary(value: Any) -> dict[str, Any]: + summary = _tensor_summary(value) + shape = summary["shape"] + if len(shape) != 2 or shape[0] != 1 or shape[1] <= 0: + raise ValueError("deterministic regression requires one batch with a nonempty target axis") + if not summary["finite"]: + raise ValueError("deterministic regression target values must be finite") + return { + "kind": "regression_values", + "target_count": shape[1], + "regression_targets": shape[1], + "parameter_elements": 0, + "values": value[0].detach().float().cpu().tolist(), + "axes": ["target"], + "target_names": [], + "target_units": [], + } + + +def _regression_summary( + output: Any, distribution: str, parameter_names: Sequence[str] +) -> dict[str, Any]: + if distribution not in {"normal", "student_t", "negative_binomial"}: + raise ValueError("regression reference requires a declared probability distribution") + if not isinstance(output, (tuple, list)) or len(output) != len(parameter_names) or not output: + raise ValueError("regression reference parameters do not match the declared distribution") + # Transformers names distribution parameters loc/df. The public Task + # contract names them location/degrees_of_freedom; never infer tuple order. + aliases = {"loc": "location", "df": "degrees_of_freedom"} + names = [aliases.get(name, name) for name in parameter_names] + required = { + "normal": {"location", "scale"}, + "student_t": {"degrees_of_freedom", "location", "scale"}, + "negative_binomial": {"total_count", "logits"}, + }[distribution] + if len(names) != len(required) or set(names) != required: + raise ValueError( + "regression reference parameter names do not match the declared distribution" + ) + parameters = [] + targets = None + for name, value in zip(names, output): + shape = tuple(int(size) for size in value.shape) + if ( + len(shape) != 2 + or shape[0] != 1 + or shape[1] <= 0 + or (targets is not None and shape[1] != targets) + ): + raise ValueError( + "regression parameters must each have one batch and the same target axis" + ) + targets = shape[1] + if not _tensor_summary(value)["finite"]: + raise ValueError("regression reference returned nonfinite parameters") + parameters.append({"name": name, "values": value[0].detach().float().cpu().tolist()}) + return { + "distribution": distribution, + "target_count": targets, + "regression_targets": targets, + "parameter_elements": targets * len(parameters), + "axes": ["target"], + "parameters": parameters, + "target_names": [], + "target_units": [], + } + + +def _task_value( + arguments: argparse.Namespace, + request: Mapping[str, Any], + name: str, + default: Any = None, +) -> Any: + if name in request: + return request[name] + return default + + +def _load_tts( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + import torch + from transformers import AutoProcessor, BarkModel + + torch.use_deterministic_algorithms(True) + device = torch.device("cuda") + prompt = str(request.get("prompt", "")) + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + BarkModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + inputs = _to_device(processor(prompt, return_tensors="pt"), device) + seed = _request_seed(request, 42) + generation_options = _bark_generation_options(request) + + def invoke() -> Mapping[str, Any]: + _seed_all(torch, seed) + with torch.inference_mode(): + audio = model.generate(**inputs, **generation_options) + return { + "audio_samples": int(audio.numel()), + "sample_rate": int(model.generation_config.sample_rate), + "_audio_f32": audio.detach().float().cpu().reshape(-1).numpy(), + } + + return Session(invoke, "transformers") + + +def _load_asr( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + import torch + from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + read_wav_float32, + resample_audio, + ) + + audio, sample_rate = read_wav_float32(str(_asset_path(arguments, request, "audio_path"))) + target_rate = 16_000 + audio = resample_audio(audio, sample_rate, target_rate) + max_new_tokens = int(request.get("max_new_tokens", 100)) + device = torch.device("cuda") + + from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + AutoModelForSpeechSeq2Seq.from_pretrained( + arguments.model, **_load_kwargs(arguments, torch) + ) + .eval() + .to(device) + ) + processor_options: dict[str, Any] = { + "sampling_rate": target_rate, + "return_tensors": "pt", + } + language = str(request.get("language", "") or "") + if language: + processor_options["language"] = language + inputs = _to_device( + processor(audio, **processor_options), device, next(model.parameters()).dtype + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + generated = model.generate(**inputs, max_new_tokens=max_new_tokens) + sequences = generated.sequences if hasattr(generated, "sequences") else generated + token_ids = [int(token) for token in sequences[0].detach().cpu().tolist()] + return { + "text": processor.batch_decode(sequences, skip_special_tokens=True)[0], + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + return Session(invoke, "transformers") + + +def _load_vlm_model( + transformers_module: Any, class_name: str, model_id: str, kwargs: dict[str, Any] +) -> Any: + if class_name not in { + "AutoModel", + "AutoModelForCausalLM", + "AutoModelForImageTextToText", + }: + raise ValueError(f"unsupported vision-language model_class: {class_name}") + model_class = getattr(transformers_module, class_name) + return model_class.from_pretrained(model_id, **kwargs) + + +def _vl_prompt_has_image_placeholder(text: str) -> bool: + return any( + marker in text + for marker in ( + "<|image_pad|>", + "<|vision_start|>", + "<|image_1|>", + "", + "", + ) + ) + + +def _load_vlm( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + workflow = str(options.get("workflow", "chat")) + if workflow not in {"chat", "single-image-token-chat"}: + raise ValueError(f"unsupported vision-language workflow: {workflow}") + + import torch + import transformers + from PIL import Image + from transformers import AutoProcessor + + device = torch.device("cuda") + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + load_options = _load_kwargs(arguments, torch) + if bool(options.get("eager_attention", False)): + config = transformers.AutoConfig.from_pretrained( + arguments.model, **_processor_kwargs(arguments) + ) + config._attn_implementation = "eager" + config._attn_implementation_internal = "eager" + load_options.update({"config": config, "attn_implementation": "eager"}) + model = ( + _load_vlm_model( + transformers, + str(options.get("model_class", "AutoModelForImageTextToText")), + arguments.model, + load_options, + ) + .eval() + .to(device) + ) + image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") + prompt = str(request.get("prompt", "")) + max_new_tokens = int(request.get("max_new_tokens", 16)) + + if workflow == "single-image-token-chat": + messages = [{"role": "user", "content": f"<|image_1|>{prompt}"}] + template_owner = getattr(processor, "tokenizer", processor) + else: + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": prompt}, + ], + } + ] + template_owner = processor + rendered = template_owner.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + if not isinstance(rendered, str) or not _vl_prompt_has_image_placeholder(rendered): + raise ValueError("vision-language chat template lost the image placeholder") + inputs = processor(text=rendered, images=image, padding=True, return_tensors="pt") + inputs = _to_device(inputs, device, next(model.parameters()).dtype) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + output_ids = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = output_ids[0] + generated = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in generated.detach().cpu().tolist()] + text = processor.decode(generated, skip_special_tokens=True) + return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} + + return Session(invoke, "transformers") + + +def _load_embedding( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + configured = _json_object( + getattr(arguments, "timing_contract_json", "{}"), "--timing-contract-json" + ) + if not configured: + raise ValueError("embedding reference requires explicit --timing-contract-json") + declared_timing = timing_contract(runner="task-reference", declared=configured) + if declared_timing["asset_loading_included"]: + raise ValueError( + "embedding reference preloads its assets; asset_loading_included must be false" + ) + prompts = _batch_prompt(request) + if len(prompts) != 1: + raise ValueError("embedding reference requires exactly one text input") + import torch + from transformers import AutoModel, AutoTokenizer + + device = torch.device("cuda") + tokenizer = AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + AutoModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + compile_evidence = _compile_forward(model) if arguments.mode == "torch-compile" else None + prompt = prompts[0] + + def prepare_inputs() -> Mapping[str, Any]: + return _to_device( + tokenizer(prompt, return_tensors="pt", truncation=True), + device, + ) + + prepared_inputs = None + if not declared_timing["input_preparation_included"]: + prepared_inputs = prepare_inputs() + + def invoke() -> Any: + inputs = prepare_inputs() if prepared_inputs is None else prepared_inputs + with torch.inference_mode(): + outputs = model(**inputs, output_hidden_states=True) + hidden = getattr(outputs, "last_hidden_state", None) + if hidden is None: + hidden = outputs.hidden_states[-1] + mask = inputs.get("attention_mask", torch.ones(hidden.shape[:2], device=device)) + mask = mask.unsqueeze(-1).to(hidden.dtype) + vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) + vector = torch.nn.functional.normalize(vector, p=2, dim=-1) + return vector.detach().to("cpu", dtype=torch.float32) + + def summarize(vector: Any) -> Mapping[str, Any]: + summary = _tensor_summary(vector) + summary.update( + { + "embedding_vectors": 1, + "embedding_elements": int(vector.numel()), + "dim": int(vector.shape[-1]), + } + ) + return summary + + return Session( + invoke, + "transformers", + timing_scope=str(declared_timing["timing_scope"]), + input_preparation_included=bool(declared_timing["input_preparation_included"]), + asset_loading_included=False, + compile_evidence=compile_evidence, + summarize=summarize, + ) + + +def _load_reranking( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + import torch + from transformers import AutoModelForSequenceClassification, AutoProcessor + + device = torch.device("cuda") + documents = [str(document) for document in request.get("documents", [])] + query = str(request.get("query", "")) + model = ( + AutoModelForSequenceClassification.from_pretrained( + arguments.model, **_load_kwargs(arguments, torch) + ) + .eval() + .to(device) + ) + processor = AutoProcessor.from_pretrained( + arguments.model, + max_input_tiles=6, + use_thumbnail=True, + rerank_max_length=8192, + **_processor_kwargs(arguments), + ) + examples = [ + {"question": query, "doc_text": document, "doc_image": ""} for document in documents + ] + inputs = processor.process_queries_documents_crossencoder(examples) + inputs = _to_device(inputs, device) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + logits = model(**inputs).logits.detach().float().cpu() + if logits.ndim == 2: + logits = logits[:, 0] if logits.shape[-1] == 1 else logits[:, -1] + scores = [float(score) for score in logits.reshape(-1).tolist()] + return {"scores": scores, "document_count": len(documents)} + + return Session(invoke, "transformers") + + +_PIXART_TRTMC_MIXED_PRECISION = "pixart_fp16_dit_fp32_t5" + + +def _diffusion_component_precision_contract( + arguments: argparse.Namespace, + options: Mapping[str, Any], +) -> str: + contract = str(options.get("component_precision_contract", "") or "") + if not contract: + return "" + if contract != _PIXART_TRTMC_MIXED_PRECISION: + raise ValueError(f"unsupported diffusion component precision contract: {contract}") + if arguments.precision != "fp16": + raise ValueError(f"{contract} requires precision=fp16") + return contract + + +def _cast_floating_tensors(value: Any, dtype: Any, torch_module: Any) -> Any: + if isinstance(value, torch_module.Tensor): + return value.to(dtype=dtype) if value.is_floating_point() else value + if isinstance(value, tuple): + return tuple(_cast_floating_tensors(item, dtype, torch_module) for item in value) + if isinstance(value, list): + return [_cast_floating_tensors(item, dtype, torch_module) for item in value] + if isinstance(value, dict): + return { + key: _cast_floating_tensors(item, dtype, torch_module) for key, item in value.items() + } + return value + + +def _configure_diffusion_component_precision( + pipeline: Any, + arguments: argparse.Namespace, + options: Mapping[str, Any], + torch_module: Any, +) -> None: + contract = _diffusion_component_precision_contract(arguments, options) + if not contract: + return + + def fp16_transformer_inputs(_module: Any, args: Any, kwargs: Any) -> Any: + return ( + _cast_floating_tensors(args, torch_module.float16, torch_module), + _cast_floating_tensors(kwargs, torch_module.float16, torch_module), + ) + + def fp32_transformer_output(_module: Any, _args: Any, output: Any) -> Any: + return _cast_floating_tensors(output, torch_module.float32, torch_module) + + pipeline.transformer.register_forward_pre_hook(fp16_transformer_inputs, with_kwargs=True) + pipeline.transformer.register_forward_hook(fp32_transformer_output) + + +def _diffusion_pipeline( + arguments: argparse.Namespace, + torch_module: Any, + options: Mapping[str, Any], +) -> Any: + import diffusers + + model_id = str(options.get("model_id", arguments.model)) + requested_revision = ( + str(options.get("model_revision", getattr(arguments, "revision", None) or "")) or None + ) + model_source: str | Path = model_id + if arguments.local_files_only: + cached = _cached_snapshot_path(model_id, requested_revision, "model_index.json") + if cached is None: + raise FileNotFoundError(f"cached Diffusers snapshot is missing: {model_id}") + model_source = cached + + load_options: dict[str, Any] = {} + component_contract = _diffusion_component_precision_contract(arguments, options) + if component_contract == _PIXART_TRTMC_MIXED_PRECISION: + from transformers import T5EncoderModel + + load_options["text_encoder"] = T5EncoderModel.from_pretrained( + model_source, + subfolder="text_encoder", + torch_dtype=torch_module.float32, + **( + {"revision": requested_revision} + if requested_revision and model_source == model_id + else {} + ), + local_files_only=arguments.local_files_only, + ) + vae_class_name = options.get("vae_class") + if vae_class_name is not None: + if not isinstance(vae_class_name, str) or not vae_class_name.startswith("Autoencoder"): + raise ValueError("diffusion adapter_options.vae_class must name an Autoencoder class") + try: + vae_class = getattr(diffusers, vae_class_name) + except AttributeError as error: + raise ValueError(f"unsupported Diffusers VAE class: {vae_class_name}") from error + vae_precision = str(options.get("vae_precision", arguments.precision)) + load_options["vae"] = vae_class.from_pretrained( + model_source, + subfolder="vae", + torch_dtype=_torch_dtype(torch_module, vae_precision), + **( + {"revision": requested_revision} + if requested_revision and model_source == model_id + else {} + ), + local_files_only=arguments.local_files_only, + ) + return diffusers.DiffusionPipeline.from_pretrained( + model_source, + torch_dtype=_torch_dtype(torch_module, arguments.precision), + **load_options, + **( + {"revision": requested_revision} + if requested_revision and model_source == model_id + else {} + ), + trust_remote_code=bool(options.get("trust_remote_code", arguments.trust_remote_code)), + local_files_only=arguments.local_files_only, + ) + + +def _load_diffusers( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + import inspect + import torch + from PIL import Image + + pipeline = _diffusion_pipeline(arguments, torch, options) + _configure_diffusion_component_precision(pipeline, arguments, options, torch) + if bool(options.get("cpu_offload", False)) and hasattr(pipeline, "enable_model_cpu_offload"): + pipeline.enable_model_cpu_offload() + else: + pipeline.to("cuda") + signature = inspect.signature(pipeline.__call__) + accepted = set(signature.parameters) + accepts_extra = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + if "prompt" in request and "prompts" in request: + raise ValueError("reference request must not provide both prompt and prompts") + prompt = request.get("prompts", request.get("prompt", "")) + if "prompts" in request and not isinstance(prompt, list): + raise ValueError("prompts must contain one string per batch item") + if isinstance(prompt, list): + if not prompt or any(not isinstance(value, str) for value in prompt): + raise ValueError("prompt list must contain one string per batch item") + count = len(prompt) + else: + if not isinstance(prompt, str): + raise ValueError("prompt must be a string or a non-empty list of strings") + count = 1 + batch_size = request.get("batch_size", count) + if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + if isinstance(prompt, list): + if count != batch_size: + raise ValueError("prompt list must contain one string per batch item") + prompt_value: str | list[str] = list(prompt) + else: + prompt_value = [prompt] * batch_size if batch_size > 1 else prompt + values: dict[str, Any] = {"prompt": prompt_value} + negative_prompt = str(request.get("negative_prompt", "")) + if ( + negative_prompt + or "negative_prompt" in request + or bool(options.get("always_negative_prompt", False)) + ): + values["negative_prompt"] = negative_prompt + steps = int(request.get("num_steps", -1)) + if steps > 0: + values.update({"num_inference_steps": steps, "step": steps}) + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + height = int( + request.get("height", request.get("video_height", manifest.get("image_height", 0))) + ) + width = int(request.get("width", request.get("video_width", manifest.get("image_width", 0)))) + if height > 0: + values["height"] = height + if width > 0: + values["width"] = width + num_frames = int( + request.get( + "video_num_frames", + _task_value(arguments, request, "num_frames", manifest.get("video_num_frames", 1)), + ) + ) + if num_frames > 0: + values["num_frames"] = num_frames + for name in ("action", "intrinsics"): + value = str(_task_value(arguments, request, name, "") or "") + if value: + values[name] = value + fps = int(_task_value(arguments, request, "fps", 0)) + if fps > 0: + values["fps"] = fps + for name in ("flow_shift", "translation_speed", "rotation_speed_deg"): + value = _task_value(arguments, request, name) + if value is not None: + values[name] = float(value) + cfg_scale = float(request.get("cfg_scale", -1.0)) + guidance_parameter = str(options.get("guidance_parameter", "guidance_scale")) + if guidance_parameter not in {"guidance_scale", "true_cfg_scale"}: + raise ValueError(f"unsupported diffusion guidance_parameter: {guidance_parameter}") + if cfg_scale >= 0 and guidance_parameter != "true_cfg_scale": + values["cfg_scale"] = cfg_scale + if bool(request.get("no_refiner", False)): + values["no_refiner"] = True + guidance = float(request.get("guidance_scale", -1.0)) + if guidance_parameter == "true_cfg_scale": + true_cfg_scale = guidance if guidance >= 0 else cfg_scale + if true_cfg_scale >= 0: + values["true_cfg_scale"] = true_cfg_scale + elif guidance >= 0: + values["guidance_scale"] = guidance + values["output_type"] = "np" + if "image_path" in request and "image_paths" in request: + raise ValueError("reference request must not provide both image_path and image_paths") + image_path = request.get("image_path", "") + if "image_paths" in request: + image_paths = request["image_paths"] + if ( + not isinstance(image_paths, list) + or len(image_paths) != 1 + or not isinstance(image_paths[0], str) + or not image_paths[0] + ): + raise ValueError("this reference supports exactly one conditioning image") + image_path = image_paths[0] + task = getattr(arguments, "selected_task", None) or manifest["task"] + if task in {"images_text_to_image_edit", "image_edit"} and not image_path: + raise ValueError("image edit reference requires one conditioning image") + if image_path: + if "image" not in accepted and not accepts_extra: + raise ValueError("selected reference pipeline does not accept conditioning image") + values["image"] = Image.open( + _asset_path(arguments, {"image_path": image_path}, "image_path") + ).convert("RGB") + call_values = { + name: value for name, value in values.items() if name in accepted or accepts_extra + } + required = [str(name) for name in options.get("required_call_arguments", [])] + missing = [name for name in required if name not in call_values] + if missing: + raise ValueError( + f"{arguments.adapter} reference is missing required call arguments: " + + ", ".join(missing) + ) + seed = int(request.get("seed", 42)) + raw_seeds = request.get("seeds") + if raw_seeds is not None: + if not isinstance(raw_seeds, list) or len(raw_seeds) != batch_size: + raise ValueError("seeds must contain one integer per batch item") + seeds: int | list[int] = [int(value) for value in raw_seeds] + elif batch_size > 1: + seeds = [seed] * batch_size + else: + seeds = seed + if "generator" in accepted or accepts_extra: + if isinstance(seeds, list): + call_values["generator"] = [ + torch.Generator("cuda").manual_seed(value) for value in seeds + ] + else: + call_values["generator"] = torch.Generator("cuda").manual_seed(seeds) + + def invoke() -> Mapping[str, Any]: + if "generator" in call_values: + generators = call_values["generator"] + if isinstance(generators, list): + for generator, value in zip(generators, seeds, strict=True): + generator.manual_seed(value) + else: + generators.manual_seed(seeds) + result = pipeline(**call_values) + media = getattr(result, "images", None) + if media is None: + media = getattr(result, "frames", None) + media_type = str(request.get("media_type", "image")) + return {**_media_summary(media, media_type), "_media": media} + + return Session( + invoke, + "diffusers", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + ) + + +def _load_modular_diffusers( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + """Run a ModularPipeline whose workflow is declared by the family profile.""" + import torch + from diffusers import ModularPipeline + + workflow = str(options.get("workflow", "")) + output_name = str(options.get("output", "videos")) + if not workflow or not output_name: + raise ValueError("modular Diffusers requires adapter_options.workflow and output") + load_options = { + "workflow": workflow, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + load_options["revision"] = arguments.revision + pipeline = ModularPipeline.from_pretrained(arguments.model, **load_options) + pipeline.load_components( + dtype=_torch_dtype(torch, arguments.precision), + pretrained_model_name_or_path=arguments.model, + local_files_only=arguments.local_files_only, + ) + pipeline = pipeline.to("cuda") + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + seed = int(request.get("seed", 42)) + generator = torch.Generator().manual_seed(seed) + values: dict[str, Any] = { + "prompt": str(request.get("prompt", "")), + "height": int(request.get("height", manifest.get("image_height", 0))), + "width": int(request.get("width", manifest.get("image_width", 0))), + "num_inference_steps": int(request.get("num_steps", 0)), + "generator": generator, + "output": output_name, + "output_type": "np", + } + frames = int(request.get("video_num_frames", manifest.get("video_num_frames", 1))) + if frames > 1: + values["num_frames"] = frames + for source, target in ( + ("negative_prompt", "negative_prompt"), + ("guidance_scale", "guidance_scale"), + ("cfg_scale", "cfg_scale"), + ): + if source in request: + values[target] = request[source] + + def invoke() -> Mapping[str, Any]: + generator.manual_seed(seed) + media = pipeline(**values) + media_type = "video" if frames > 1 else "image" + return {**_media_summary(media, media_type), "_media": media} + + return Session( + invoke, + "diffusers-modular", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + ) + + +def _media_count(media: Any, media_type: str) -> int: + """Count image batches or video frames without coercing arrays to bool.""" + if media is None: + return 0 + try: + outer_count = len(media) + except TypeError: + return 1 + if outer_count == 0 or media_type != "video": + return outer_count + try: + return len(media[0]) + except TypeError: + return outer_count + + +def _media_summary(media: Any, media_type: str) -> dict[str, Any]: + """Describe materialized image/video geometry without copying its pixels.""" + count = _media_count(media, media_type) + item = media + try: + item = item[0] + if media_type == "video": + item = item[0] + except (IndexError, KeyError, TypeError): + pass + width = height = channels = None + size = getattr(item, "size", None) + bands = getattr(item, "getbands", None) + if isinstance(size, tuple) and len(size) == 2 and callable(bands): + width, height = (int(value) for value in size) + channels = len(bands()) + else: + shape = tuple(int(value) for value in getattr(item, "shape", ())) + if len(shape) >= 3: + if shape[-1] in {1, 3, 4}: + height, width, channels = shape[-3:] + elif shape[-3] in {1, 3, 4}: + channels, height, width = shape[-3:] + summary = { + "media_type": media_type, + "media_count": count, + "height": height, + "width": width, + "channels": channels, + } + try: + import numpy as np + + array = np.asarray(media) + if np.issubdtype(array.dtype, np.number): + finite = bool(np.isfinite(array).all()) + if not finite: + raise RuntimeError("reference returned non-finite media values") + summary["finite"] = True + except (TypeError, ValueError): + pass + return summary + + +def _media_items(media: Any, media_type: str) -> list[Any]: + """Return image items, or frames from the first generated video.""" + import numpy as np + + if media_type == "video": + if isinstance(media, np.ndarray): + values = media[0] if media.ndim == 5 else media + else: + values = media[0] if isinstance(media, (list, tuple)) and media else media + else: + values = media + if isinstance(values, np.ndarray): + if values.ndim == 3: + return [values] + return [values[index] for index in range(len(values))] + if isinstance(values, (list, tuple)): + return list(values) + return [values] + + +def _write_media_artifacts(summary: dict[str, Any], output: Path) -> None: + """Materialize deterministic sample frames after reference timing completes.""" + import numpy as np + from PIL import Image + + media = summary.pop("_media") + items = _media_items(media, str(summary.get("media_type", "image"))) + declared = int(summary.get("media_count", 0)) + if declared != len(items) or declared < 1: + raise RuntimeError("reference media count differs from its materialized output") + indices = sorted({0, declared // 2, declared - 1}) + directory = output.with_suffix(".media").resolve() + directory.mkdir(parents=True, exist_ok=True) + artifacts = [] + for index in indices: + item = items[index] + values = np.asarray(item) + scale = 255.0 if np.issubdtype(values.dtype, np.floating) else 1.0 + image = ( + item + if isinstance(item, Image.Image) + else Image.fromarray(np.clip(values * scale, 0, 255).astype(np.uint8)) + ) + path = directory / f"{index:06d}.png" + image.convert("RGB").save(path) + artifacts.append(str(path)) + summary["artifact_indices"] = indices + summary["frame_artifacts" if summary.get("media_type") == "video" else "image_artifacts"] = ( + artifacts + ) + + +def _numeric_values(request: Mapping[str, Any], key: str) -> list[float]: + raw = request.get(key) + if not isinstance(raw, list) or not raw: + raise ValueError(f"time-series reference requires non-empty request.{key}") + return [float(value) for value in raw] + + +def _observed_values(request: Mapping[str, Any], count: int) -> list[float]: + raw = request.get("observed_mask") + if raw is None or raw == []: + return [1.0] * count + values = _numeric_values(request, "observed_mask") + if len(values) != count: + raise ValueError("time-series observed_mask must match past_values") + return values + + +def _align(values: Sequence[float], length: int, fill: float) -> list[float]: + result = [fill] * length + count = min(len(values), length) + result[-count:] = values[-count:] + return result + + +def _patchtst_task(config: Any) -> str: + for attribute in ("patchtst_task", "task_type", "problem_type"): + value = str(getattr(config, attribute, "") or "").lower() + if "class" in value: + return "classification" + if "regress" in value: + return "regression" + if "forecast" in value or "predict" in value: + return "forecast" + + architectures = getattr(config, "architectures", []) or [] + if isinstance(architectures, str): + architectures = [architectures] + for architecture in architectures: + value = str(architecture).lower() + if "class" in value: + return "classification" + if "regress" in value: + return "regression" + if "forecast" in value or "predict" in value: + return "forecast" + return "regression" + + +def _load_timeseries( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + import torch + import transformers + + device = torch.device("cuda") + dtype = _torch_dtype(torch, arguments.precision) + task_id = _selected_task(arguments) + reference_type = str(options.get("reference_type", "")) + if reference_type == "chronos-bolt": + from chronos import ChronosBoltPipeline + + chronos_options = _processor_kwargs(arguments) + chronos_options.update({"device_map": str(device), "dtype": dtype}) + model = ChronosBoltPipeline.from_pretrained(arguments.model, **chronos_options) + compile_evidence = None + if arguments.mode == "torch-compile": + compile_evidence = _compile_forward(model.model) + raw = _numeric_values(request, "past_values") + observed = _observed_values(request, len(raw)) + context = torch.tensor( + [value if mask > 0 else float("nan") for value, mask in zip(raw, observed)], + dtype=dtype, + device=device, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + value = model.predict( + context, + prediction_length=model.model_prediction_length, + limit_prediction_length=True, + ) + quantiles = model.model.config.chronos_config["quantiles"] + return _forecast_summary(value, task_id, quantiles) + + return Session(invoke, "chronos", compile_evidence=compile_evidence) + + config = transformers.AutoConfig.from_pretrained( + arguments.model, **_processor_kwargs(arguments) + ) + if reference_type == "timesfm": + model = ( + transformers.TimesFmModelForPrediction.from_pretrained( + arguments.model, **_load_kwargs(arguments, torch) + ) + .eval() + .to(device) + ) + length = int(model.config.context_length) + raw = _numeric_values(request, "past_values") + series = torch.tensor(_align(raw, length, 0.0), dtype=dtype, device=device).reshape( + 1, length + ) + observed = _align(_observed_values(request, len(raw)), length, 0.0) + padding = [0 if mask > 0 else 1 for mask in observed] + padding_tensor = torch.tensor(padding, dtype=torch.int32, device=device).reshape(1, length) + frequency = int(request.get("frequency", 0)) + frequency_tensor = torch.tensor([[frequency]], dtype=torch.long, device=device) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + decoder = model.decoder( + past_values=series, + past_values_padding=padding_tensor, + freq=frequency_tensor, + output_attentions=False, + output_hidden_states=False, + ) + output = model._postprocess_output( + decoder.last_hidden_state, (decoder.loc, decoder.scale) + )[:, -1, : model.config.horizon_length, 0] + return _forecast_summary(output, task_id) + + else: + if reference_type not in {"patchtsmixer", "patchtst"}: + raise ValueError(f"unsupported time-series reference_type: {reference_type}") + is_mixer = reference_type == "patchtsmixer" + if is_mixer: + model_class = transformers.PatchTSMixerForPrediction + output_name = "prediction_outputs" + else: + task = _patchtst_task(config) + class_name, output_name = { + "classification": ( + "PatchTSTForClassification", + "prediction_logits", + ), + "forecast": ("PatchTSTForPrediction", "prediction_outputs"), + "regression": ("PatchTSTForRegression", "regression_outputs"), + }[task] + model_class = getattr(transformers, class_name) + if not hasattr(model_class, "all_tied_weights_keys"): + model_class.all_tied_weights_keys = {} + model = ( + model_class.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + length = int(config.context_length) + channels = int(config.num_input_channels) + raw = _numeric_values(request, "past_values") + values = torch.tensor( + _align(raw, length * channels, 0.0), dtype=dtype, device=device + ).reshape(1, length, channels) + aligned_mask = _align(_observed_values(request, len(raw)), length * channels, 0.0) + observed = torch.tensor( + aligned_mask, + dtype=dtype, + device=device, + ).reshape(1, length, channels) + + def invoke() -> Mapping[str, Any]: + with ( + torch.inference_mode(), + torch.autocast( + device_type="cuda", + dtype=dtype, + enabled=dtype != torch.float32, + ), + ): + if is_mixer: + outputs = model( + past_values=values * observed, + observed_mask=observed, + return_loss=False, + return_dict=True, + ) + output = outputs.prediction_outputs + else: + outputs = model( + past_values=values, + past_observed_mask=observed.gt(0.5), + return_dict=True, + ) + output = getattr(outputs, output_name) + if task_id == "series_to_regression_distribution": + return _regression_summary( + output, config.distribution_output, tuple(model.distribution_output.args_dim) + ) + if task_id == "series_to_regression_values": + return _regression_values_summary(output) + if isinstance(output, (tuple, list)): + output = torch.stack(list(output), dim=-1) + return _forecast_summary(output, task_id) + + return Session(invoke, "transformers") + + +def _load_vision( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + import torch + from PIL import Image + import transformers + + device = torch.device("cuda") + image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") + width, height = image.size + kwargs = _load_kwargs(arguments, torch) + processor_kwargs = _processor_kwargs(arguments) + vision_task = str(options.get("vision_task", "")) + + if arguments.adapter == "timm-classification": + import timm + from timm.data import create_transform, resolve_model_data_config + + model_id = arguments.model + if arguments.revision: + model_id += "@" + arguments.revision + model = timm.create_model(f"hf-hub:{model_id}", pretrained=True) + dtype = _torch_dtype(torch, arguments.precision) + model = model.eval().to(device=device, dtype=dtype) + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image).unsqueeze(0).to(device=device, dtype=dtype) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + logits = model(inputs) + return {"top_class": int(logits.argmax(dim=-1)[0]), **_tensor_summary(logits)} + + elif vision_task == "object-detection": + processor = transformers.AutoImageProcessor.from_pretrained( + arguments.model, **processor_kwargs + ) + model = ( + transformers.AutoModelForObjectDetection.from_pretrained(arguments.model, **kwargs) + .eval() + .to(device) + ) + inputs = _to_device( + processor(images=image, return_tensors="pt"), + device, + next(model.parameters()).dtype, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + target_sizes = torch.tensor([[height, width]], device=device) + results = processor.post_process_object_detection( + outputs, threshold=0.5, target_sizes=target_sizes + )[0] + return { + "detected_images": 1, + "detections": int(results["scores"].shape[0]), + "image_height": height, + "image_width": width, + "boxes": results["boxes"].float().cpu().reshape(-1).tolist(), + "scores": results["scores"].float().cpu().tolist(), + "class_ids": results["labels"].cpu().tolist(), + "shape": [int(results["scores"].shape[0]), 4], + "coordinates": "xyxy", + "units": "pixels", + } + + elif vision_task == "image-features": + processor = transformers.AutoImageProcessor.from_pretrained( + arguments.model, **processor_kwargs + ) + model = transformers.AutoModel.from_pretrained(arguments.model, **kwargs).eval().to(device) + inputs = _to_device( + processor(images=image, return_tensors="pt"), + device, + next(model.parameters()).dtype, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + return { + "last_hidden_state_shape": _tensor_summary(outputs.last_hidden_state)["shape"], + "pooler_output_shape": _tensor_summary(outputs.pooler_output)["shape"], + } + + elif vision_task == "semantic-segmentation": + processor = transformers.AutoImageProcessor.from_pretrained( + arguments.model, **processor_kwargs + ) + model = ( + transformers.AutoModelForSemanticSegmentation.from_pretrained(arguments.model, **kwargs) + .eval() + .to(device) + ) + inputs = _to_device( + processor(images=image, return_tensors="pt"), + device, + next(model.parameters()).dtype, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(height, width)] + )[0] + return { + "num_masks": 1, + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + } + + else: + raise ValueError(f"unsupported Transformers vision_task: {vision_task}") + + framework = "timm" if arguments.adapter == "timm-classification" else "transformers" + return Session(invoke, framework) + + + +LOADERS: dict[ + str, Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session] +] = { + "hf-diffusers": _load_diffusers, + "hf-diffusers-modular": _load_modular_diffusers, + "hf-transformers-asr": _load_asr, + "hf-transformers-embedding": _load_embedding, + "hf-transformers-reranking": _load_reranking, + "hf-transformers-tts": _load_tts, + "hf-transformers-vision": _load_vision, + "hf-transformers-vlm": _load_vlm, + "pytorch-timeseries": _load_timeseries, + "timm-classification": _load_vision, +} + + +def _synchronize() -> None: + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.synchronize() + except ImportError: + return + + +def _compile_forward(model: Any) -> dict[str, Any]: + import torch + from torch._dynamo.backends.registry import lookup_backend + + evidence = {"compiled_graph_count": 0} + inductor = lookup_backend("inductor") + + def compile_graph(graph: Any, inputs: Any, **options: Any) -> Any: + compiled = inductor(graph, inputs, **options) + evidence["compiled_graph_count"] += 1 + return compiled + + model.forward = torch.compile( + model.forward, + backend=compile_graph, + fullgraph=False, + dynamic=False, + ) + evidence.update( + { + "api": "torch.compile", + "target": "model.forward", + "backend": "inductor", + "mode": "default", + "fullgraph": False, + "dynamic": False, + "applied": True, + } + ) + return evidence + + +def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: + output: Any = {} + untimed_iterations = max(warmup, int(session.compile_evidence is not None)) + for _ in range(untimed_iterations): + output = session.invoke() + _synchronize() + compiled_graphs = None + if session.compile_evidence is not None: + compiled_graphs = int(session.compile_evidence["compiled_graph_count"]) + if compiled_graphs < 1: + raise RuntimeError("warmup did not execute a compiled graph") + samples = [] + for _ in range(iterations): + _synchronize() + started = time.perf_counter() + output = session.invoke() + _synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + if ( + compiled_graphs is not None + and int(session.compile_evidence["compiled_graph_count"]) != compiled_graphs + ): + raise RuntimeError("model compilation occurred inside timed samples") + summary = session.summarize(output) if session.summarize is not None else output + return samples, dict(summary) + + +def _environment() -> dict[str, Any]: + value: dict[str, Any] = { + "platform": platform.platform(), + "python": platform.python_version(), + "python_executable": sys.executable, + } + try: + import torch + + value["torch"] = torch.__version__ + if torch.cuda.is_available(): + value["gpu"] = torch.cuda.get_device_name(0) + value["cuda"] = torch.version.cuda + except ImportError: + pass + for module_name in ("transformers", "diffusers", "nemo", "chronos", "moshi"): + try: + module = __import__(module_name) + except ImportError: + continue + value[module_name] = str(getattr(module, "__version__", "unknown")) + return value + + +def run(arguments: argparse.Namespace) -> int: + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + options = _json_object(arguments.adapter_options_json, "--adapter-options-json") + expected_mode = "pytorch-eager" if arguments.adapter in PYTORCH_ADAPTERS else "hf-eager" + supported_modes = {expected_mode} + if ( + arguments.adapter == "pytorch-timeseries" + and options.get("reference_type") == "chronos-bolt" + ): + supported_modes.add("torch-compile") + if arguments.adapter == "hf-transformers-embedding": + supported_modes.add("torch-compile") + if arguments.mode not in supported_modes: + raise ValueError(f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}") + request = flatten_config(_json_object(arguments.request_json, "--request-json")) + configured_timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + fields = ("timing_scope", "input_preparation_included", "asset_loading_included") + expected_timing = None + if configured_timing: + if set(configured_timing) != set(fields): + raise ValueError( + "--timing-contract-json must declare exactly the three reference timing fields" + ) + declared = timing_contract(runner="task-reference", declared=configured_timing) + expected_timing = {name: declared[name] for name in fields} + load_started = time.perf_counter() + load_seconds: float | None = None + compile_evidence: dict[str, Any] | None = None + session = LOADERS[arguments.adapter](arguments, request, options) + compile_evidence = session.compile_evidence + load_seconds = time.perf_counter() - load_started + framework = session.framework + timing_scope = session.timing_scope + input_included = session.input_preparation_included + asset_included = session.asset_loading_included + actual_timing = { + "timing_scope": timing_scope, + "input_preparation_included": input_included, + "asset_loading_included": asset_included, + } + timing_contract(runner="task-reference", declared=actual_timing) + if expected_timing is not None and actual_timing != expected_timing: + raise RuntimeError( + f"{arguments.adapter} reference implementation timing drifted: " + f"actual={actual_timing}, declared={expected_timing}" + ) + samples, output_summary = _measure(session, arguments.warmup, arguments.iterations) + if "_media" in output_summary: + _write_media_artifacts(output_summary, arguments.output) + disparity = output_summary.pop("_disparity_f32", None) + if disparity is not None: + artifact_path = arguments.output.with_suffix(".disparity.f32").resolve() + artifact_path.parent.mkdir(parents=True, exist_ok=True) + disparity.tofile(artifact_path) + output_summary["disparity_artifact"] = str(artifact_path) + audio = output_summary.pop("_audio_f32", None) + if audio is not None: + import soundfile as sf + + artifact_path = arguments.output.with_suffix(".audio.wav").resolve() + artifact_path.parent.mkdir(parents=True, exist_ok=True) + sf.write(artifact_path, audio, output_summary["sample_rate"], subtype="FLOAT") + output_summary["audio_artifact"] = str(artifact_path) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": arguments.adapter, + "adapter": arguments.adapter, + "framework": framework, + "mode": arguments.mode, + "precision": arguments.precision, + "padding": arguments.padding, + "experts_implementation": None, + "compile_scope": "model.forward" if arguments.mode == "torch-compile" else None, + "compile_evidence": compile_evidence, + "timing_scope": timing_scope, + "input_preparation_included": input_included, + "asset_loading_included": asset_included, + "model_load_included": False, + "model_load_seconds": load_seconds, + "model": arguments.model, + "case_name": arguments.case_name, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + "timing_scope": timing_scope, + "input_preparation_included": input_included, + "asset_loading_included": asset_included, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": statistics.median(samples), + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": _environment(), + "finished_at": datetime.now(timezone.utc).isoformat(), + } + if compile_evidence is not None: + compile_evidence["warmup_completed"] = True + compile_evidence["timed_callable_uses_compiled_target"] = True + if not all(math.isfinite(float(value)) and float(value) > 0.0 for value in samples): + raise RuntimeError("reference produced an invalid timing sample") + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + arguments = build_parser().parse_args(argv) + if arguments.local_files_only: + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + return run(arguments) + except (KeyError, OSError, RuntimeError, TypeError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/benchmark/performance/baselines/hf_transformers.py b/qualification_tests/benchmark_qualification/performance/references/hf_transformers.py similarity index 100% rename from apps/benchmark/performance/baselines/hf_transformers.py rename to qualification_tests/benchmark_qualification/performance/references/hf_transformers.py diff --git a/apps/benchmark/performance/baselines/timing_contracts.py b/qualification_tests/benchmark_qualification/performance/references/timing_contracts.py similarity index 100% rename from apps/benchmark/performance/baselines/timing_contracts.py rename to qualification_tests/benchmark_qualification/performance/references/timing_contracts.py diff --git a/qualification_tests/benchmark_qualification/performance/reporting.py b/qualification_tests/benchmark_qualification/performance/reporting.py new file mode 100644 index 0000000000..144ebe7cbf --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/reporting.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-independent JSON and HTML reporting for Performance qualification.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import html +import json +import math +from pathlib import Path +from typing import Any, Mapping + +from .types import PerfMatrixError + + +RESULT_SCHEMA = "trtmc.perf-matrix/v2" +REPORT_SCHEMA = "trtmc.perf-report/v2" + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_report( + run_directory: Path, + results: Mapping[str, Any], + preparation: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + source_schema = results.get("schema_version", RESULT_SCHEMA) + row_key = "cases" if source_schema == "trtmc.perf-matrix/v1" else "rows" + rows = [dict(row) for row in results.get(row_key, []) if isinstance(row, Mapping)] + selected_ids = results.get("selected_entry_ids") + if not isinstance(selected_ids, list) or not all( + isinstance(value, str) for value in selected_ids + ): + raise PerfMatrixError("matrix results selected_entry_ids must be strings") + completed_ids = {str(row.get("id")) for row in rows} + counts = { + status: sum(row.get("status") == status for row in rows) + for status in ("green", "yellow", "red", "contract-mismatch", "white") + } + report = { + "schema_version": REPORT_SCHEMA, + "source_schema_version": source_schema, + "generated_at": _now(), + "status": results.get("status", "unknown"), + "suite": results.get("suite"), + "environment": results.get("environment"), + "summary": { + "selected": len(selected_ids), + "pending": sum(entry_id not in completed_ids for entry_id in selected_ids), + "comparable": counts["green"] + counts["yellow"] + counts["red"], + **counts, + }, + "rows": rows, + } + if preparation is not None: + report["preparation"] = dict(preparation) + _write_json(run_directory / "report.json", report) + (run_directory / "report.html").write_text(_report_html(report), encoding="utf-8") + return report + + +def _measurement_html(value: Any) -> str: + if not isinstance(value, Mapping): + return "Not recorded" + metrics = value.get("metrics", {}) + latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} + samples = value.get("samples_ms", []) + measured = ( + sorted( + float(sample) + for sample in samples + if isinstance(sample, (int, float)) + and not isinstance(sample, bool) + and math.isfinite(sample) + and sample > 0 + ) + if isinstance(samples, list) + else [] + ) + parts = [] + for name, percentile in (("p50", 0.5), ("p95", 0.95)): + number = latency.get(name) if isinstance(latency, Mapping) else None + if number is None and measured: + position = (len(measured) - 1) * percentile + lower, upper = math.floor(position), math.ceil(position) + number = measured[lower] + (measured[upper] - measured[lower]) * (position - lower) + if ( + isinstance(number, (int, float)) + and not isinstance(number, bool) + and math.isfinite(number) + ): + parts.append(f"{name}: {number:,.3f} ms") + policy = value.get("measurement_policy", {}) + scope = value.get("timing_scope") or ( + policy.get("timing_scope") if isinstance(policy, Mapping) else None + ) + parts.append("Scope: " + html.escape(str(scope or "not recorded"))) + if isinstance(metrics, Mapping): + parts.extend( + f"{html.escape(name)}: {number:,.3f}" + for name, number in metrics.items() + if name.endswith("_per_s") + and isinstance(number, (int, float)) + and not isinstance(number, bool) + and math.isfinite(number) + ) + return "
".join(parts) + + +def _report_html(report: Mapping[str, Any]) -> str: + rows = [] + for row in report.get("rows", []): + comparison = row.get("comparison", {}) if isinstance(row, Mapping) else {} + measurement_stability = ( + row.get("measurement_stability", {}) if isinstance(row, Mapping) else {} + ) + candidate = _measurement_html( + row.get("candidate") + or {"metrics": {"latency_ms": {"p50": comparison.get("candidate_p50_ms")}}} + ) + reference = _measurement_html( + row.get("reference", row.get("baseline")) + or {"metrics": {"latency_ms": {"p50": comparison.get("reference_p50_ms")}}} + ) + reason = comparison.get("reason", row.get("error", "")) + stability = ( + measurement_stability.get("status", "") + if isinstance(measurement_stability, Mapping) + else "" + ) + evidence = { + key: row[key] + for key in ( + "commands", + "comparison", + "measurement_stability", + "resolved_settings", + "baseline_contract", + ) + if key in row + } + detail = html.escape(json.dumps(evidence, indent=2, sort_keys=True)) + link = "" + artifact = row.get("artifact_dir") + if ( + isinstance(artifact, str) + and artifact + and not Path(artifact).is_absolute() + and ".." not in Path(artifact).parts + and not any(value in artifact for value in (":", "\\")) + ): + from urllib.parse import quote + + link = f'Case artifacts' + rows.append( + "" + f"{html.escape(str(row.get('id', '')))}" + f"{html.escape(str(row.get('model', '')))}" + f"{html.escape(str(row.get('operation', '')))}" + f"{html.escape(str(row.get('status', '')))}" + f"{html.escape(str(stability))}" + f"{candidate}" + f"{reference}" + f"{html.escape(str(reason))}" + f"
Evidence / commands{link}
{detail}
" + "" + ) + return f""" +TRTMC performance + +

TRTMC performance matrix

+

Source data: {html.escape(str(report.get("source_schema_version", RESULT_SCHEMA)))}. Original timing scopes and recorded outcomes are preserved.

+

Status: {html.escape(str(report.get("status", "unknown")))}; +selected: {html.escape(str(report.get("summary", {}).get("selected", 0)))}; +pending: {html.escape(str(report.get("summary", {}).get("pending", 0)))}

+ + + +{"".join(rows)}
EntryModelOperationStatusStabilityCandidate measurementsReference measurementsReasonEvidence
+

Green is faster by more than the margin, yellow is within the margin, red is +slower by more than the margin, and white is not comparable.

+

Output comparison follows the suite's declared contract; matching dimensions or token counts alone does not establish model correctness. Use family correctness tests for acceptance.

+

Original results · Report data

+ + +""" diff --git a/qualification_tests/benchmark_qualification/performance/runner.py b/qualification_tests/benchmark_qualification/performance/runner.py new file mode 100644 index 0000000000..277a657813 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/runner.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Execute one internal Performance Qualification Target.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +from typing import Any, Mapping + +from . import matrix + + +def run_case( + spec: Mapping[str, Any], + environment: matrix.Environment, + output: Path, + *, + no_build: bool, + verbose: bool, +) -> dict[str, Any]: + """Run one case through the same execution contract as a campaign matrix.""" + + resolved = matrix.preflight([dict(spec)], environment, require_runtime=True) + if len(resolved) != 1: + raise matrix.PerfMatrixError("single-case qualification resolved an unexpected case count") + output.mkdir(parents=True, exist_ok=True) + entry = resolved[0] + try: + result = matrix.execute_entry( + entry, + environment, + output, + no_build=no_build, + verbose=verbose, + attempt=1, + ) + except (OSError, matrix.PerfMatrixError, subprocess.SubprocessError) as error: + result = { + "id": str(entry.spec["id"]), + "model": entry.model.name, + "family": entry.model.family, + "operation": entry.spec["operation"], + "testcase": entry.case.testcase_name, + "status": "white", + "attempts": 1, + "error": str(error), + } + receipt = { + "schema_version": matrix.RESULT_SCHEMA, + "status": ( + "completed" if result.get("status") in matrix.TERMINAL_COMPARISONS else "failed" + ), + "suite": str(spec["id"]), + "environment": environment.name, + "selected_entry_ids": [str(spec["id"])], + "rows": [result], + } + (output / "results.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + matrix.write_report(output, receipt) + return result diff --git a/qualification_tests/benchmark_qualification/performance/stability.py b/qualification_tests/benchmark_qualification/performance/stability.py new file mode 100644 index 0000000000..910ed18a45 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/stability.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-independent timing stability policy.""" + +from __future__ import annotations + +import math +import statistics +from typing import Any, Mapping, Sequence + +from .types import PerfMatrixError + + +SAMPLE_COUNT = 10 +MAX_HALF_MEDIAN_CHANGE_PERCENT = 5.0 +MEDIAN_BAND_PERCENT = 5.0 +MINIMUM_SAMPLES_IN_BAND = 8 + + +def p50(value: Mapping[str, Any]) -> float: + metrics = value.get("metrics", {}) + latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} + value_p50 = latency.get("p50") if isinstance(latency, Mapping) else None + if ( + isinstance(value_p50, bool) + or not isinstance(value_p50, (int, float)) + or not math.isfinite(float(value_p50)) + ): + raise PerfMatrixError("measurement has no finite latency p50") + return float(value_p50) + + +def timing_stability(values: Sequence[Any]) -> dict[str, Any]: + if len(values) != SAMPLE_COUNT: + return { + "status": "not_evaluated", + "sample_count": len(values), + "reason": "requires_10_samples", + } + try: + samples = [float(value) for value in values] + except (TypeError, ValueError): + return { + "status": "not_evaluated", + "sample_count": len(values), + "reason": "invalid_samples", + } + if not all(math.isfinite(value) and value > 0.0 for value in samples): + return { + "status": "not_evaluated", + "sample_count": len(samples), + "reason": "invalid_samples", + } + middle = len(samples) // 2 + median_ms = statistics.median(samples) + first_half_median_ms = statistics.median(samples[:middle]) + second_half_median_ms = statistics.median(samples[middle:]) + half_change_percent = ( + abs(second_half_median_ms - first_half_median_ms) / first_half_median_ms * 100.0 + ) + samples_within_band = sum( + abs(sample - median_ms) / median_ms * 100.0 <= MEDIAN_BAND_PERCENT for sample in samples + ) + stable = ( + half_change_percent <= MAX_HALF_MEDIAN_CHANGE_PERCENT + and samples_within_band >= MINIMUM_SAMPLES_IN_BAND + ) + return { + "status": "stable" if stable else "unstable", + "sample_count": len(samples), + "median_ms": median_ms, + "first_half_median_ms": first_half_median_ms, + "second_half_median_ms": second_half_median_ms, + "half_median_change_percent": half_change_percent, + "samples_within_band": samples_within_band, + } + + +def measurement_stability( + reference: Mapping[str, Any], candidate: Mapping[str, Any] +) -> dict[str, Any]: + reference_result = timing_stability(reference.get("samples_ms", [])) + candidate_result = timing_stability(candidate.get("samples_ms", [])) + statuses = {reference_result["status"], candidate_result["status"]} + if statuses == {"stable"}: + status = "stable" + elif "unstable" in statuses: + status = "unstable" + else: + status = "not_evaluated" + return { + "status": status, + "policy": { + "required_samples": SAMPLE_COUNT, + "max_half_median_change_percent": MAX_HALF_MEDIAN_CHANGE_PERCENT, + "median_band_percent": MEDIAN_BAND_PERCENT, + "minimum_samples_within_band": MINIMUM_SAMPLES_IN_BAND, + }, + "reference": reference_result, + "candidate": candidate_result, + } diff --git a/qualification_tests/benchmark_qualification/performance/tests/__init__.py b/qualification_tests/benchmark_qualification/performance/tests/__init__.py new file mode 100644 index 0000000000..0eb8c14ecc --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/tests/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Performance Qualification contract tests.""" diff --git a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py b/qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py similarity index 92% rename from apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py rename to qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py index 19f67a0757..06a1d0ee46 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py @@ -14,7 +14,7 @@ import numpy as np import pytest -from apps.benchmark.performance.baselines import task_reference +from qualification_tests.benchmark_qualification.performance.references import generic_reference @pytest.fixture @@ -164,7 +164,7 @@ def arguments(class_name="FluxPipeline", family="flux", task="text_to_image", ** def test_prompt_and_seed_order_reaches_every_pipeline_invocation(recording_diffusers, payload, expected_prompt, expected_seeds): recorded = recording_diffusers source = deepcopy(payload) - session = task_reference._load_diffusers(recorded.arguments(), task_reference.flatten_config(payload), {}) + session = generic_reference._load_diffusers(recorded.arguments(), generic_reference.flatten_config(payload), {}) summaries = [session.invoke() for _ in range(3)] assert len(recorded.loads) == len(recorded.constructors) == 1 assert len(recorded.calls) == 3 @@ -182,10 +182,10 @@ def test_prompt_and_seed_order_reaches_every_pipeline_invocation(recording_diffu def test_measurement_resets_each_seed_after_warmups(recording_diffusers): recorded = recording_diffusers - session = task_reference._load_diffusers( + session = generic_reference._load_diffusers( recorded.arguments(), {"prompt": ["second", "first"], "seeds": [2**32 - 1, 0]}, {}, ) - samples, summary = task_reference._measure(session, warmup=2, iterations=3) + samples, summary = generic_reference._measure(session, warmup=2, iterations=3) assert len(samples) == 3 assert len(recorded.calls) == 5 assert all(call["observed_seeds"] == [2**32 - 1, 0] for call in recorded.calls) @@ -212,7 +212,7 @@ def test_measurement_resets_each_seed_after_warmups(recording_diffusers): def test_invalid_batch_requests_do_not_call_pipeline(recording_diffusers, payload, match): recorded = recording_diffusers with pytest.raises(ValueError, match=match): - task_reference._load_diffusers(recorded.arguments(), payload, {}) + generic_reference._load_diffusers(recorded.arguments(), payload, {}) assert recorded.calls == [] @@ -225,9 +225,9 @@ def test_qwen_edit_checkpoint_selects_edit_class_and_receives_full_request(recor image.write_bytes(b"P6\n1 1\n255\n\xff\x00\x80") request = {"prompt": "Make it blue", image_key: [image.name] if image_key == "image_paths" else image.name, "config": {"seed": 2**32 - 1, "num_steps": 5, "guidance_scale": 3.25, "negative_prompt": " "}} - session = task_reference._load_diffusers( + session = generic_reference._load_diffusers( arguments, - task_reference.flatten_config(request), + generic_reference.flatten_config(request), { "cpu_offload": True, "always_negative_prompt": True, @@ -265,7 +265,7 @@ def test_qwen_edit_checkpoint_selects_edit_class_and_receives_full_request(recor def test_qwen_native_guidance_controls_true_cfg_and_preserves_empty_negative_prompt(recording_diffusers, controls, expected, negative): recorded = recording_diffusers arguments = recorded.arguments("QwenImagePipeline", "synthetic") - session = task_reference._load_diffusers( + session = generic_reference._load_diffusers( arguments, {"prompt": "a", **negative, **controls}, {"always_negative_prompt": True, "guidance_parameter": "true_cfg_scale"}, @@ -278,7 +278,7 @@ def test_qwen_native_guidance_controls_true_cfg_and_preserves_empty_negative_pro def test_non_qwen_guidance_retains_provider_parameter(recording_diffusers): recorded = recording_diffusers - task_reference._load_diffusers(recorded.arguments(), {"prompt": "a", "guidance_scale": 1.5}, {}).invoke() + generic_reference._load_diffusers(recorded.arguments(), {"prompt": "a", "guidance_scale": 1.5}, {}).invoke() assert recorded.calls[0]["guidance_scale"] == 1.5 assert recorded.constructors[0].devices == ["cuda"] @@ -296,7 +296,7 @@ def test_qwen_edit_image_contract_fails_before_invocation(recording_diffusers, i "QwenImageEditPlusPipeline", "synthetic", "images_text_to_image_edit" ) with pytest.raises(ValueError, match=match): - task_reference._load_diffusers(arguments, {"prompt": "edit", **inputs}, {}) + generic_reference._load_diffusers(arguments, {"prompt": "edit", **inputs}, {}) assert recorded.calls == recorded.images == [] @@ -304,7 +304,7 @@ def test_conditioning_image_is_never_silently_filtered(recording_diffusers): recorded = recording_diffusers arguments = recorded.arguments("QwenImagePipeline", "synthetic", "images_text_to_image_edit") with pytest.raises(ValueError, match="does not accept conditioning image"): - task_reference._load_diffusers(arguments, {"prompt": "edit", "image_paths": ["condition.ppm"]}, {}) + generic_reference._load_diffusers(arguments, {"prompt": "edit", "image_paths": ["condition.ppm"]}, {}) assert recorded.calls == [] @@ -335,7 +335,7 @@ def test_checkpoint_owned_loader_preserves_component_precision_and_revision( "model_revision": "override-revision", "trust_remote_code": True, } - session = task_reference._load_diffusers(arguments, {"prompt": "a"}, options) + session = generic_reference._load_diffusers(arguments, {"prompt": "a"}, options) session.invoke() source, kwargs, component = recorded.components[0] expected_source = Path(arguments.model) if local_files_only else arguments.model @@ -356,10 +356,10 @@ def test_model_id_override_and_missing_local_snapshot_keep_load_contract(recordi recorded = recording_diffusers arguments = recorded.arguments() override = recorded.arguments("Flux2Pipeline").model - pipeline = task_reference._diffusion_pipeline(arguments, recorded.torch, {"model_id": override}) + pipeline = generic_reference._diffusion_pipeline(arguments, recorded.torch, {"model_id": override}) assert pipeline.__class__.__name__ == "Flux2Pipeline" assert recorded.loads[0][1] == Path(override) - monkeypatch.setattr(task_reference, "_cached_snapshot_path", lambda *_: None) + monkeypatch.setattr(generic_reference, "_cached_snapshot_path", lambda *_: None) with pytest.raises(FileNotFoundError, match="cached Diffusers snapshot is missing"): - task_reference._diffusion_pipeline(arguments, recorded.torch, {}) + generic_reference._diffusion_pipeline(arguments, recorded.torch, {}) assert len(recorded.loads) == 1 diff --git a/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py b/qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py similarity index 97% rename from apps/benchmark/trtmc_benchmark/tests/test_family_performance.py rename to qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py index e959c40910..bbe9017ff4 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py @@ -15,7 +15,8 @@ import pytest import yaml -import tools.perf_matrix as perf +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance.references import generic_reference REFERENCE_SCRIPT = '''"""Synthetic reference fixture; no model or GPU qualification.""" @@ -78,6 +79,22 @@ def invoke(): ''' +def test_generic_reference_has_no_family_dispatch() -> None: + source = Path(generic_reference.__file__).read_text(encoding="utf-8").casefold() + for family in ( + "magpie_tts", + "canary", + "nemotron_speech_streaming", + "glmasr", + "detr", + "dinov3", + "segformer", + "sam3", + ): + assert family not in source + assert "arguments.family" not in source + + def test_custom_script_default_mode_does_not_claim_torch_compilation(): assert perf._baseline_mode({"script": "tests/reference.py"}) == "reference" assert perf._baseline_mode({"script": "tests/reference.py", "mode": "pytorch-eager"}) == "pytorch-eager" @@ -125,7 +142,7 @@ def repository(tmp_path, monkeypatch): (folder / "manifests" / (owner + ".json")).write_text(json.dumps(manifest)) script = families / "sample/tests/performance_reference.py" script.write_text(REFERENCE_SCRIPT, encoding="utf-8") - canonical = root / "apps/benchmark/performance/release.yaml" + canonical = root / "qualification_tests/benchmark_qualification/performance/config/release.yaml" _write_suite(canonical, [_entry("other", "other-model")]) owned_suite = families / "sample/tests/performance.yaml" _write_suite(owned_suite, [_entry("sample", "sample-model", script=True)]) @@ -810,9 +827,9 @@ def test_execute_entry_validates_actual_script_process(repository, tmp_path, mon if failure: expected = "reference command failed" if failure == "process" else "case_name" with pytest.raises(perf.PerfMatrixError, match=expected): - perf._execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) + perf.execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) else: - row = perf._execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) + row = perf.execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) assert row["reference"]["output_summary"]["loaded_models"] == 1 assert row["reference"]["output_summary"]["invocations"] == 5 # The existing ten-sample performance gate is intentionally unchanged. diff --git a/apps/benchmark/trtmc_benchmark/tests/test_offline_speech_contract.py b/qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py similarity index 99% rename from apps/benchmark/trtmc_benchmark/tests/test_offline_speech_contract.py rename to qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py index a0afa46ad5..8fa76b1f42 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_offline_speech_contract.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py @@ -9,7 +9,7 @@ import pytest -import tools.perf_matrix as perf +from qualification_tests.benchmark_qualification.performance import matrix as perf def _entry(task="offline_speech_dialogue", *, primary="duplex_speech_dialogue", explicit=False): diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py similarity index 57% rename from apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py rename to qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py index d042fc2997..892b52133c 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py @@ -17,22 +17,20 @@ import pytest import yaml -import tools.perf_matrix as perf -from apps.benchmark.performance.baselines import ( +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance.references import ( hf_transformers, - lance_reference, - sana_wm_reference, - task_reference, + generic_reference, ) from apps.benchmark.trtmc_benchmark.types import ModelDescriptor REPO = Path(__file__).resolve().parents[4] -SUITE = REPO / "apps/benchmark/performance/release.yaml" +SUITE = REPO / "qualification_tests/benchmark_qualification/performance/config/release.yaml" def test_shared_reference_runner_never_uses_family_identity_to_select_behavior() -> None: - tree = ast.parse(Path(task_reference.__file__).read_text(encoding="utf-8")) + tree = ast.parse(Path(generic_reference.__file__).read_text(encoding="utf-8")) parents: dict[ast.AST, ast.AST] = { child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent) } @@ -129,15 +127,15 @@ def __call__(self, prompt, height=None, width=None): pil = ModuleType("PIL") pil.Image = SimpleNamespace() monkeypatch.setitem(sys.modules, "PIL", pil) - monkeypatch.setattr(task_reference, "_diffusion_pipeline", lambda *_: Pipeline() if accepts_frames else ImagePipeline()) + monkeypatch.setattr(generic_reference, "_diffusion_pipeline", lambda *_: Pipeline() if accepts_frames else ImagePipeline()) original = perf.ManifestCatalog(REPO / "families").resolve(selector) model = replace(original, testcases=({**original.testcases[0], **controls},)) case = perf.resolve_case(model, tmp_path / "model.bundle", selected_task=task) candidate_request = json.loads(json.dumps(case.request)) - request = task_reference.flatten_config(case.request) + request = generic_reference.flatten_config(case.request) before = dict(request) arguments = SimpleNamespace(manifest=model.manifest_path, family=model.family, precision=model.precision) - task_reference._load_diffusers(arguments, request, {}).invoke() + generic_reference._load_diffusers(arguments, request, {}).invoke() wanted_height, wanted_width, wanted_frames = expected if expected is not None else (height, width, frames) assert captured["height"] == wanted_height assert captured["width"] == wanted_width @@ -404,14 +402,14 @@ def test_reference_selection_does_not_rewrite_manifest_or_silently_choose_defaul manifest.write_text('{"task":"series_to_point_forecast"}') original = manifest.read_bytes() arguments = SimpleNamespace(manifest=manifest, selected_task="series_to_quantile_forecast") - assert task_reference._selected_task(arguments) == "series_to_quantile_forecast" + assert generic_reference._selected_task(arguments) == "series_to_quantile_forecast" assert manifest.read_bytes() == original arguments.selected_task = None - assert task_reference._selected_task(arguments) == "series_to_point_forecast" + assert generic_reference._selected_task(arguments) == "series_to_point_forecast" for invalid in ("", " ", " series_to_point_forecast", 7): arguments.selected_task = invalid with pytest.raises(ValueError, match="selected_task"): - task_reference._selected_task(arguments) + generic_reference._selected_task(arguments) @pytest.mark.parametrize("payload", [{"token_ids": []}, {"token_ids": [7], "prompt": "hello"}]) @@ -423,7 +421,7 @@ def test_text_only_reference_loaders_never_replace_token_input_with_empty_text(p "asset_loading_included": False, })) with pytest.raises(ValueError, match="token_ids"): - task_reference._load_embedding(arguments, payload, {}) + generic_reference._load_embedding(arguments, payload, {}) class _SummaryTensor: @@ -463,23 +461,23 @@ def tolist(self): def test_forecast_reference_names_axes_without_arbitrary_squeeze_or_sorting(): - point = task_reference._forecast_summary( + point = generic_reference._forecast_summary( _SummaryTensor(np.zeros((1, 4, 3))), "series_to_point_forecast" ) assert point["shape"] == [4, 3] and point["axes"] == ["horizon", "channel"] assert point["horizon_steps"] == [1, 2, 3, 4] and point["forecast_elements"] == 12 - quantile = task_reference._forecast_summary( + quantile = generic_reference._forecast_summary( _SummaryTensor(np.zeros((1, 3, 4))), "series_to_quantile_forecast", [0.1, 0.5, 0.9] ) assert quantile["shape"] == [3, 4, 1] assert quantile["axes"] == ["quantile", "horizon", "channel"] assert quantile["quantile_levels"] == [0.1, 0.5, 0.9] with pytest.raises(ValueError, match="one-series"): - task_reference._forecast_summary( + generic_reference._forecast_summary( _SummaryTensor(np.zeros((2, 4, 3))), "series_to_point_forecast" ) with pytest.raises(ValueError, match="quantile levels"): - task_reference._forecast_summary( + generic_reference._forecast_summary( _SummaryTensor(np.zeros((1, 3, 4))), "series_to_quantile_forecast", [] ) @@ -520,7 +518,7 @@ def test_semantic_forecast_comparison_rejects_swapped_or_missing_axes(): def test_regression_values_reference_retains_single_target_axis(): - summary = task_reference._regression_values_summary(_SummaryTensor([[1.5, -2.0]])) + summary = generic_reference._regression_values_summary(_SummaryTensor([[1.5, -2.0]])) assert summary["values"] == [1.5, -2.0] assert summary["axes"] == ["target"] and summary["target_count"] == 2 assert summary["target_names"] == [] and summary["target_units"] == [] @@ -538,9 +536,9 @@ def test_regression_values_reference_retains_single_target_axis(): {"output_summary": dict(invalid)})[0] for shape in ((2,), (2, 2), (1, 0), (1, 2, 1)): with pytest.raises(ValueError, match="one batch"): - task_reference._regression_values_summary(_SummaryTensor(np.zeros(shape))) + generic_reference._regression_values_summary(_SummaryTensor(np.zeros(shape))) with pytest.raises(ValueError, match="finite"): - task_reference._regression_values_summary(_SummaryTensor([[float("inf")]])) + generic_reference._regression_values_summary(_SummaryTensor([[float("inf")]])) def test_head_score_performance_contract_preserves_shape_and_representation(): @@ -579,7 +577,7 @@ def test_head_score_performance_contract_rejects_incomplete_outputs(changed): def test_regression_distribution_keeps_parameter_names_and_target_axis(): values = (_SummaryTensor([[1.0, 2.0]]), _SummaryTensor([[0.5, 0.7]])) - summary = task_reference._regression_summary(values, "normal", ["loc", "scale"]) + summary = generic_reference._regression_summary(values, "normal", ["loc", "scale"]) assert summary["target_count"] == 2 and summary["axes"] == ["target"] assert summary["parameters"] == [ {"name": "location", "values": [1.0, 2.0]}, @@ -595,10 +593,10 @@ def test_regression_distribution_keeps_parameter_names_and_target_axis(): {"output_summary": {**summary, "distribution": "student_t"}}, )[0] with pytest.raises(ValueError, match="same target axis"): - task_reference._regression_summary( + generic_reference._regression_summary( (_SummaryTensor([[1, 2]]), _SummaryTensor([[1]])), "normal", ["loc", "scale"] ) - student = task_reference._regression_summary( + student = generic_reference._regression_summary( (_SummaryTensor([[4.0]]), _SummaryTensor([[1.0]]), _SummaryTensor([[0.5]])), "student_t", ["df", "loc", "scale"], ) @@ -606,7 +604,7 @@ def test_regression_distribution_keeps_parameter_names_and_target_axis(): "degrees_of_freedom", "location", "scale" ] with pytest.raises(ValueError, match="parameter names"): - task_reference._regression_summary(values, "normal", ["loc", "location"]) + generic_reference._regression_summary(values, "normal", ["loc", "location"]) @pytest.mark.parametrize("distribution,parameters", [ @@ -662,635 +660,13 @@ def test_regression_distribution_order_fix_retains_mismatch_rejection(changed): def test_timeseries_reference_preserves_observed_masks_and_unobserved_padding(): request = {"past_values": [1, 2, 3], "observed_mask": [1, 0, 1]} - observed = task_reference._observed_values(request, 3) + observed = generic_reference._observed_values(request, 3) assert observed == [1.0, 0.0, 1.0] - assert task_reference._align(observed, 5, 0.0) == [0, 0, 1, 0, 1] - assert task_reference._observed_values({"past_values": [1, 2]}, 2) == [1, 1] - assert task_reference._observed_values({"observed_mask": []}, 2) == [1, 1] + assert generic_reference._align(observed, 5, 0.0) == [0, 0, 1, 0, 1] + assert generic_reference._observed_values({"past_values": [1, 2]}, 2) == [1, 1] + assert generic_reference._observed_values({"observed_mask": []}, 2) == [1, 1] with pytest.raises(ValueError, match="match past_values"): - task_reference._observed_values({"observed_mask": [1]}, 2) - - -def _voicechat_reference_fixture(tmp_path, monkeypatch): - events = [] - reference = tmp_path / "speech-reference" - utility = reference / "nemo/collections/speechlm2/inference/utils/offline_voicechat.py" - utility.parent.mkdir(parents=True) - utility.write_text("# official utility location fixture\n", encoding="utf-8") - checkpoint = tmp_path / "checkpoint" - checkpoint.mkdir() - (checkpoint / "config.json").write_text("{}", encoding="utf-8") - audio_path = tmp_path / "request.wav" - audio_path.write_bytes(b"fixture; decoded by the mocked official loader") - manifest = tmp_path / "manifest.json" - manifest.write_text('{"task":"offline_speech_dialogue"}', encoding="utf-8") - arguments = SimpleNamespace( - model=str(checkpoint), - revision="checkpoint-revision", - manifest=manifest, - family="nemotron_voicechat", - precision="fp32", - operation="speech_dialogue", - local_files_only=True, - ) - request = {"audio_path": str(audio_path)} - options = {"reference_repo": str(reference)} - state = { - "kwargs": [], - "prompts": [], - "events": events, - "metadata": SimpleNamespace(samplerate=16000, channels=1, frames=4), - } - - class Tensor(_SummaryTensor): - def __init__(self, values, label="output"): - super().__init__(values) - self.label = label - - def to(self, device): - events.append(("transfer", self.label, device)) - return self - - def __getitem__(self, key): - return Tensor(self.values[key], self.label) - - def cpu(self): - events.append(("materialize", self.label)) - return self - - def numpy(self): - return np.asarray(self.values, dtype=np.float32) - - class Model: - source_sample_rate = 16000 - target_sample_rate = 22050 - - def float(self): - events.append("float32") - return self - - model = Model() - signal = Tensor([[0.1, 0.2, 0.3, 0.4]], "signal") - lengths = Tensor([4], "lengths") - upstream = ModuleType("nemo.collections.speechlm2.inference.utils.offline_voicechat") - upstream.__file__ = str(utility) - - def build_model(path, *, device): - events.append(("build", path, device)) - return model - - def load_wav(path, *, device): - events.append(("decode", path, device)) - return signal, signal, lengths - - def encode_prompt(received_model, prompt, *, device): - assert received_model is model - events.append(("prompt", prompt, device)) - state["prompts"].append(prompt) - return "prompt-token-buffer", "prompt-length-buffer" - - def inference(received_model, **kwargs): - assert received_model is model - events.append("infer") - state["kwargs"].append(kwargs) - # The padded NaN must not be included in the valid audio result. - return state.get( - "result", - { - "text": ["Full spoken answer."], - "audio": Tensor([[0.1, 0.2, 0.3, float("nan")]]), - "audio_len": Tensor([3]), - }, - ) - - upstream.build_model = build_model - upstream.load_wav_16k_mono = load_wav - upstream.encode_system_prompt = encode_prompt - upstream.run_offline_inference = inference - sf = ModuleType("soundfile") - sf.info = lambda _path: state["metadata"] - - def write(path, values, rate, **kwargs): - events.append(("write_audio", str(path), rate, kwargs)) - state["written_audio"] = np.array(values, copy=True) - - sf.write = write - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setitem(sys.modules, upstream.__name__, upstream) - monkeypatch.setitem(sys.modules, "soundfile", sf) - monkeypatch.setitem(sys.modules, "torch", ModuleType("torch")) - monkeypatch.setattr( - task_reference, "_seed_all", lambda _torch, seed: events.append(("seed", seed)) - ) - monkeypatch.setattr(task_reference, "_synchronize", lambda: events.append("barrier")) - return arguments, request, options, state, upstream, Tensor - - -def test_voicechat_reference_builds_once_and_times_real_offline_calls(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - session = task_reference._load_voicechat(arguments, request, options) - assert session.timing_scope == "task-pipeline-call-wall" - assert session.input_preparation_included is True and session.asset_loading_included is False - assert state["events"] == [ - ("build", arguments.model, "cuda"), - "float32", - ("decode", request["audio_path"], "cpu"), - ] - samples, output = task_reference._measure(session, warmup=1, iterations=2) - assert len(samples) == 2 and all(value > 0 for value in samples) - assert sum(isinstance(event, tuple) and event[0] == "build" for event in state["events"]) == 1 - assert sum(isinstance(event, tuple) and event[0] == "decode" for event in state["events"]) == 1 - assert state["events"].count("infer") == 3 - assert state["events"].count(("seed", 0)) == 3 - assert state["events"].count(("transfer", "signal", "cuda")) == 3 - assert state["prompts"] == [task_reference.VOICECHAT_SYSTEM_PROMPT] * 3 - assert output["text"] == "Full spoken answer." - assert output["audio_samples"] == output["num_samples"] == 3 - assert output["sample_rate"] == 22050 and output["channels"] == 1 - assert output["input_samples"] == 4 and output["input_sample_rate"] == 16000 - np.testing.assert_array_equal( - output["_audio_f32"], np.asarray([0.1, 0.2, 0.3], dtype=np.float32) - ) - for kwargs in state["kwargs"]: - assert kwargs["decode_audio"] is True and kwargs["input_pad_len"] == 0 - assert kwargs["prompt_tokens"] == "prompt-token-buffer" - assert kwargs["prompt_token_lens"] == "prompt-length-buffer" - assert not ( - {"temperature", "top_p", "repetition_penalty", "function_calls"} & kwargs.keys() - ) - - -def test_voicechat_reference_preserves_explicit_prompt_sampling_and_seed(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - session = task_reference._load_voicechat( - arguments, - { - **request, - "system_prompt": " An explicit spoken prompt. ", - "config": { - "seed": 17, - "temperature": 0.0, - "top_p": 1.0, - "repetition_penalty": 1.2, - "presence_penalty": 0.0, - }, - }, - options, - ) - session.invoke() - kwargs = state["kwargs"][0] - assert state["prompts"] == [" An explicit spoken prompt. "] - assert ("seed", 17) in state["events"] - assert { - key: kwargs[key] - for key in ("temperature", "top_p", "repetition_penalty", "presence_penalty") - } == {"temperature": 0.0, "top_p": 1.0, "repetition_penalty": 1.2, "presence_penalty": 0.0} - - -@pytest.mark.parametrize( - "patch", - [ - {"tools": []}, - {"tail_frames": 1}, - {"finish_tail_frames": -1}, - {"max_new_tokens": 256}, - {"system_prompt": " "}, - {"seed": -1}, - {"temperature": True}, - ], -) -def test_voicechat_reference_rejects_unmatched_scope_before_loading(tmp_path, monkeypatch, patch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - with pytest.raises(ValueError): - task_reference._load_voicechat(arguments, {**request, **patch}, options) - assert state["events"] == [] - - -def test_voicechat_reference_does_not_relabel_live_dialogue_as_offline(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - arguments.manifest.write_text('{"task":"duplex_speech_dialogue"}', encoding="utf-8") - with pytest.raises(ValueError, match="live or tool"): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == [] - - -def test_voicechat_reference_uses_explicit_offline_task_without_changing_primary(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - arguments.manifest.write_text('{"task":"duplex_speech_dialogue"}', encoding="utf-8") - original = arguments.manifest.read_bytes() - arguments.selected_task = "offline_speech_dialogue" - session = task_reference._load_voicechat(arguments, request, options) - output = session.invoke() - assert state["kwargs"] and isinstance(output["text"], str) - assert output["audio_samples"] == output["num_samples"] - assert arguments.manifest.read_bytes() == original - for task in ("duplex_speech_dialogue", "tool_speech_dialogue", "unknown_task", ""): - arguments.selected_task = task - before = list(state["events"]) - with pytest.raises(ValueError): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == before - - -def test_voicechat_empty_prompt_matches_native_default_selection(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch) - session = task_reference._load_voicechat(arguments, {**request, "system_prompt": ""}, options) - session.invoke() - assert state["prompts"] == [task_reference.VOICECHAT_SYSTEM_PROMPT] - - -def test_voicechat_reference_rejects_unmatched_audio_and_stale_import(tmp_path, monkeypatch): - arguments, request, options, state, upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - state["metadata"] = SimpleNamespace(samplerate=48000, channels=2, frames=4) - with pytest.raises(ValueError, match="16 kHz mono"): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == [] - upstream.__file__ = str(tmp_path / "another-installation.py") - with pytest.raises(RuntimeError, match="different NeMo installation"): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == [] - - -def test_voicechat_reference_rejects_invalid_live_output(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - session = task_reference._load_voicechat(arguments, request, options) - state["result"] = { - "text": ["real text"], - "audio": tensor([[0.1, 0.2]]), - "audio_len": tensor([3]), - } - with pytest.raises(RuntimeError, match="audio_len"): - session.invoke() - state["result"] = { - "text": ["real text"], - "audio": tensor([[float("nan")]]), - "audio_len": tensor([1]), - } - with pytest.raises(RuntimeError, match="nonfinite"): - session.invoke() - - -def test_voicechat_runner_writes_audio_after_measurement(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - arguments.adapter = "nemo-voicechat" - arguments.request_json = json.dumps(request) - arguments.adapter_options_json = json.dumps(options) - arguments.timing_contract_json = json.dumps( - { - "timing_scope": "task-pipeline-call-wall", - "input_preparation_included": True, - "asset_loading_included": False, - } - ) - arguments.mode = "pytorch-eager" - arguments.padding = "longest" - arguments.warmup, arguments.iterations = 1, 2 - arguments.case_name = "offline-voicechat" - arguments.output = tmp_path / "reference.json" - monkeypatch.setattr(task_reference, "_environment", lambda: {}) - assert task_reference.run(arguments) == 0 - result = json.loads(arguments.output.read_text(encoding="utf-8")) - assert result["model_load_included"] is False - assert result["measurement_policy"]["input_preparation_included"] is True - assert result["output_summary"]["text"] == "Full spoken answer." - assert result["output_summary"]["audio_artifact"].endswith("reference.audio.wav") - assert "_audio_f32" not in result["output_summary"] - assert state["events"][-1] == ( - "write_audio", - str(tmp_path / "reference.audio.wav"), - 22050, - {"subtype": "FLOAT"}, - ) - np.testing.assert_array_equal( - state["written_audio"], np.asarray([0.1, 0.2, 0.3], dtype=np.float32) - ) - - -def _moge_reference_fixture(tmp_path, monkeypatch): - events = [] - reference = tmp_path / "moge-reference" - module_path = reference / "moge/model/v2.py" - module_path.parent.mkdir(parents=True) - module_path.write_text("# upstream location fixture\n", encoding="utf-8") - checkpoint = tmp_path / "checkpoint" - checkpoint.mkdir() - (checkpoint / "model.pt").write_bytes(b"fixture weights") - image_path = tmp_path / "input.png" - image_path.write_bytes(b"fixture decoded by mocked PIL") - manifest = tmp_path / "manifest.json" - manifest.write_text('{"task":"image_to_metric_geometry"}', encoding="utf-8") - arguments = SimpleNamespace( - model=str(checkpoint), - revision="checkpoint-revision", - manifest=manifest, - family="moge", - operation="geometry", - precision="fp32", - local_files_only=True, - ) - state = { - "events": events, - "inputs": [], - "inference": [], - "pixels": np.asarray([[[255, 0, 127], [0, 255, 64]]], dtype=np.uint8), - "checkpoint": {"model_config": {"encoder": "fixture"}, "model": {"weight": "fixture"}}, - "arrays": { - "points": np.asarray([[[0.25, -0.5, 2], [np.inf, np.inf, np.inf]]], dtype=np.float32), - "depth": np.asarray([[2, np.inf]], dtype=np.float32), - "mask": np.asarray([[True, False]]), - "intrinsics": np.asarray([[0.8, 0, 0.5], [0, 1.1, 0.5], [0, 0, 1]], dtype=np.float32), - }, - } - - class Tensor: - def __init__(self, values): - self.values = np.asarray(values) - - def permute(self, *axes): - events.append(("permute", axes)) - return Tensor(self.values.transpose(axes)) - - def to(self, device): - events.append(("input_transfer", device)) - return self - - def detach(self): - return self - - def cpu(self): - events.append("materialize") - return self - - def numpy(self): - return self.values - - class Model: - def __init__(self, **kwargs): - events.append(("construct", kwargs)) - state["model"] = self - - def load_state_dict(self, weights, *, strict): - events.append(("load_state", weights, strict)) - return state.get("state_mismatch", ([], [])) - - def eval(self): - events.append("eval") - return self - - def float(self): - events.append("float32") - return self - - def to(self, device): - events.append(("model_transfer", device)) - return self - - def infer(self, tensor, **kwargs): - events.append("infer") - state["inputs"].append(np.array(tensor.values, copy=True)) - state["inference"].append(kwargs) - return {name: Tensor(value) for name, value in state["arrays"].items()} - - module = ModuleType("moge.model.v2") - module.__file__ = str(module_path) - module.MoGeModel = Model - attention = ModuleType("torch.nn.attention") - attention.SDPBackend = SimpleNamespace(MATH=object()) - - class MathContext: - def __enter__(self): - events.append("math_enter") - - def __exit__(self, *_args): - events.append("math_exit") - - def sdpa_kernel(backends): - assert backends == [attention.SDPBackend.MATH] - events.append("math_backend") - return MathContext() - - attention.sdpa_kernel = sdpa_kernel - fake_torch = ModuleType("torch") - - def load(path, **kwargs): - events.append(("load_checkpoint", str(path), kwargs)) - return state["checkpoint"] - - def from_numpy(values): - events.append("prepare_tensor") - return Tensor(values) - - fake_torch.load = load - fake_torch.from_numpy = from_numpy - fake_torch.backends = SimpleNamespace( - cuda=SimpleNamespace(matmul=SimpleNamespace(allow_tf32=True)), - cudnn=SimpleNamespace(allow_tf32=True), - ) - - class Image: - def __enter__(self): - return self - - def __exit__(self, *_args): - events.append("close_image") - - def convert(self, mode): - events.append(("color", mode)) - return self - - def __array__(self, dtype=None, copy=None): - return np.asarray(state["pixels"], dtype=dtype) - - def open_image(path): - events.append(("decode_image", str(path))) - return Image() - - pil = ModuleType("PIL") - pil.Image = SimpleNamespace(open=open_image) - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setitem(sys.modules, "torch", fake_torch) - monkeypatch.setitem(sys.modules, "PIL", pil) - monkeypatch.setitem(sys.modules, module.__name__, module) - monkeypatch.setitem(sys.modules, attention.__name__, attention) - monkeypatch.setattr(task_reference, "_synchronize", lambda: events.append("barrier")) - return ( - arguments, - {"image_path": str(image_path)}, - {"reference_repo": str(reference), "num_tokens": 1800}, - state, - ) - - -def test_moge_reference_uses_public_upstream_apis_and_loads_once(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - session = task_reference._load_moge(arguments, request, options) - assert session.timing_scope == "task-pipeline-call-wall" - assert session.input_preparation_included and not session.asset_loading_included - assert ( - "load_checkpoint", - str(Path(arguments.model) / "model.pt"), - {"map_location": "cpu", "weights_only": True, "mmap": True}, - ) in state["events"] - assert state["model"].onnx_compatible_mode is True - assert "prepare_tensor" not in state["events"] and "infer" not in state["events"] - samples, summary = task_reference._measure(session, warmup=1, iterations=2) - assert len(samples) == 2 and all(value > 0 for value in samples) - assert ( - sum(isinstance(event, tuple) and event[0] == "load_checkpoint" for event in state["events"]) - == 1 - ) - assert ( - sum(isinstance(event, tuple) and event[0] == "decode_image" for event in state["events"]) - == 1 - ) - assert state["events"].count("prepare_tensor") == 3 - assert ( - state["events"].count("infer") - == state["events"].count("math_backend") - == state["events"].count("math_enter") - == state["events"].count("math_exit") - == 3 - ) - for controls in state["inference"]: - assert controls == { - "num_tokens": 1800, - "use_fp16": False, - "force_projection": True, - "apply_mask": True, - } - expected = (state["pixels"].astype(np.float32) / 255.0).transpose(2, 0, 1) - for values in state["inputs"]: - np.testing.assert_array_equal(values, expected) - assert summary["point_shape"] == [1, 2, 3] and summary["units"] == "meters" - assert summary["requested_fov_x"] is None - assert np.isposinf(summary["_geometry_arrays"]["depth"][0, 1]) - - -def test_moge_reference_preserves_explicit_fov_degrees(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - session = task_reference._load_moge(arguments, {**request, "config": {"fov_x": 72.5}}, options) - summary = session.invoke() - assert state["inference"][0]["fov_x"] == 72.5 - assert summary["requested_fov_x"] == 72.5 - - -@pytest.mark.parametrize( - "request_update,options_update", - [ - ({}, {"num_tokens": None}), - ({"num_tokens": 3600}, {}), - ({"num_tokens": 1800}, {"num_tokens": 3600}), - ({"fov_x": 0}, {}), - ({"fov_x": 180}, {}), - ({"fov_x": True}, {}), - ({"apply_mask": False}, {}), - ], -) -def test_moge_reference_rejects_unqualified_controls_before_model_load( - tmp_path, monkeypatch, request_update, options_update -): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - with pytest.raises(ValueError): - task_reference._load_moge( - arguments, {**request, **request_update}, {**options, **options_update} - ) - assert not any( - isinstance(event, tuple) and event[0] == "load_checkpoint" for event in state["events"] - ) - - -def test_moge_reference_preserves_strict_checkpoint_contract(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - state["state_mismatch"] = (["uninitialized_weight"], []) - with pytest.raises(ValueError, match="state mismatch"): - task_reference._load_moge(arguments, request, options) - assert "infer" not in state["events"] - state["checkpoint"]["unexpected"] = 1 - with pytest.raises(ValueError, match="top-level"): - task_reference._load_moge(arguments, request, options) - - -def test_moge_geometry_artifacts_preserve_masked_infinity_and_calibration(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - summary = dict(task_reference._load_moge(arguments, request, options).invoke()) - task_reference._write_geometry_artifacts(summary, tmp_path / "reference.json") - assert "_geometry_arrays" not in summary - assert summary["height"] == 1 and summary["width"] == 2 and summary["valid_pixels"] == 1 - np.testing.assert_array_equal( - np.fromfile(summary["points_artifact"], dtype=" tuple[Path, perf.Environment]: @@ -1492,7 +868,7 @@ def test_shared_hf_cache_cannot_be_deleted(tmp_path: Path) -> None: def test_checked_in_environments_have_no_dead_gpu_headroom_setting() -> None: - root = REPO / "apps/benchmark/performance/environments" + root = REPO / "qualification_tests/benchmark_qualification/performance/config/environments" for path in root.glob("*.yaml"): value = yaml.safe_load(path.read_text(encoding="utf-8")) assert "minimum_gpu_free_fraction" not in value["execution"], path @@ -1725,7 +1101,7 @@ def __call__(self, **_kwargs): monkeypatch.setitem(sys.modules, "transformers", fake_transformers) compile_evidence = {"compiled_graph_count": 0} monkeypatch.setattr( - task_reference, + generic_reference, "_compile_forward", lambda _model: calls.append("compile") or compile_evidence, ) @@ -1744,7 +1120,7 @@ def __call__(self, **_kwargs): }), ) - session = task_reference.LOADERS["hf-transformers-embedding"]( + session = generic_reference.LOADERS["hf-transformers-embedding"]( arguments, {"prompt": "The quick brown fox"}, {}, @@ -1776,9 +1152,9 @@ def invoke(): compile_evidence["compiled_graph_count"] = 1 return {"value": invocations} - monkeypatch.setattr(task_reference, "_synchronize", lambda: None) - samples, output = task_reference._measure( - task_reference.Session( + monkeypatch.setattr(generic_reference, "_synchronize", lambda: None) + samples, output = generic_reference._measure( + generic_reference.Session( invoke=invoke, framework="test", compile_evidence=compile_evidence, @@ -1846,7 +1222,7 @@ def run_command(arguments, **kwargs): return successful(arguments, **kwargs) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -1882,7 +1258,7 @@ def test_compiled_output_mismatch_tries_the_configured_fallback( monkeypatch.setattr(perf, "run_command", successful) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -1902,39 +1278,6 @@ def test_compiled_output_mismatch_tries_the_configured_fallback( assert row["reference_attempts"][0]["fallback_reason"] == "output contract mismatch" -def test_lerobot_reference_is_family_owned_and_has_a_closed_contract(tmp_path: Path) -> None: - _, environment = _environment(tmp_path) - source = Path(environment.references["lerobot_repo"]) - entrypoint = source / "lerobot/common/policies/act/modeling_act.py" - entrypoint.parent.mkdir(parents=True) - entrypoint.write_text("", encoding="utf-8") - # Exercise the builtin adapter's parser and source-root contract explicitly. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "lerobot_act.control"] - resolved = perf.resolve_entries(selected, environment)[0] - command = perf.baseline_command(resolved, environment, tmp_path / "reference.json") - parsed = task_reference.build_parser().parse_args(command[2:]) - assert parsed.adapter == "pytorch-lerobot-act" - assert json.loads(parsed.adapter_options_json) == {"source_root": str(source)} - - candidate = { - "action_steps": 100, - "action_dim": 14, - "action_values": 1400, - "within_training_bounds": True, - } - reference = { - "action_steps": 100, - "action_dim": 14, - "action_values": 1400, - "finite": True, - } - assert perf._output_contract( - resolved, - {"output_summary": candidate}, - {"output_summary": reference}, - ) == (True, "", None) - def test_comparison_preserves_output_gate_and_three_performance_states( tmp_path: Path, @@ -2005,7 +1348,7 @@ def test_unstable_measurement_is_retried_once( candidate_samples=(falling, second), ) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -2043,7 +1386,7 @@ def test_scratch_is_run_scoped_and_success_cleans_all_entry_attempts( state, run_command = _fake_measurement_runner(environment, entry) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, run, @@ -2068,7 +1411,7 @@ def test_existing_artifact_attempt_is_skipped_in_one_execution(tmp_path: Path, m _, run_command = _fake_measurement_runner(environment, entry) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, run, @@ -2103,7 +1446,7 @@ def execute(_entry, _environment, _run, *, attempt, **_kwargs): "selected_entry_ids": ["first"], "rows": [], } - monkeypatch.setattr(perf, "_execute_entry", execute) + monkeypatch.setattr(perf, "execute_entry", execute) monkeypatch.setattr(perf, "_write_json", lambda *_args, **_kwargs: None) monkeypatch.setattr(perf, "write_report", lambda *_args, **_kwargs: {}) @@ -2151,7 +1494,7 @@ def test_second_measurement_contract_mismatch_discards_first_stability( ) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -2183,7 +1526,7 @@ def test_failed_remeasurement_is_not_treated_as_a_pass(tmp_path: Path, monkeypat monkeypatch.setattr(perf, "run_command", run_command) with pytest.raises(perf.PerfMatrixError, match="candidate command failed"): - perf._execute_entry( + perf.execute_entry( entry, environment, tmp_path / "run", @@ -2213,7 +1556,7 @@ def test_delete_always_cleans_declared_bundle_when_candidate_process_fails( monkeypatch.setattr(perf, "run_command", run_command) with pytest.raises(perf.PerfMatrixError, match="candidate command failed"): - perf._execute_entry( + perf.execute_entry( entry, environment, tmp_path / "run", @@ -2237,7 +1580,7 @@ def test_delete_always_cleans_declared_bundle_when_candidate_process_fails( ) monkeypatch.setattr(perf, "run_command", run_command) with pytest.raises(perf.PerfMatrixError, match="candidate command failed"): - perf._execute_entry( + perf.execute_entry( external_entry, environment, tmp_path / "external-run", @@ -2344,7 +1687,7 @@ def write_json(path, value): if path.name == "results.json": snapshots.append([row["id"] for row in value["rows"]]) - monkeypatch.setattr(perf, "_execute_entry", execute) + monkeypatch.setattr(perf, "execute_entry", execute) monkeypatch.setattr(perf, "_write_json", write_json) monkeypatch.setattr(perf, "write_report", lambda *_args, **_kwargs: {}) @@ -2372,7 +1715,7 @@ def test_contract_mismatch_is_finished_but_keeps_run_non_green(tmp_path: Path, m } monkeypatch.setattr( perf, - "_execute_entry", + "execute_entry", lambda *_args, **_kwargs: pytest.fail("finished contract mismatch was rerun"), ) monkeypatch.setattr(perf, "_write_json", lambda *_args, **_kwargs: None) @@ -2503,65 +1846,18 @@ def run_command(arguments, *, stdout_path, stderr_path, **_kwargs): def test_reference_runner_dependencies_are_baseline_owned() -> None: - root = REPO / "apps/benchmark/performance/baselines" + root = REPO / "qualification_tests/benchmark_qualification/performance/references" required = { "audio_reference.py", - "elf_reference.py", - "lance_reference.py", - "reference_support.py", - "sana_wm_reference.py", } assert all((root / name).is_file() for name in required) - source = (root / "task_reference.py").read_text(encoding="utf-8") + source = (root / "generic_reference.py").read_text(encoding="utf-8") assert "from tools." not in source assert 'REPOSITORY / "tools/' not in source assert "tests/e2e" not in source assert "tensorrt_model_connect.families" not in source -def test_lance_command_requires_explicit_checkout_without_a_commit_gate( - tmp_path: Path, -) -> None: - _, environment = _environment(tmp_path) - checkout = Path(environment.references["lance_repo"]) - (checkout / "inference_lance.py").write_text("", encoding="utf-8") - # This checkout requirement belongs to the existing builtin Lance adapter. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "lance.generate"] - resolved = perf.resolve_entries(selected, environment)[0] - command = perf.baseline_command(resolved, environment, tmp_path / "reference.json") - parsed = task_reference.build_parser().parse_args(command[2:]) - options = json.loads(parsed.adapter_options_json) - assert parsed.adapter == "upstream-lance" - assert options == { - "model_subdir": "Lance_3B", - "reference_repo": str(checkout), - "resolution": "image_768res", - "vit_subdir": "Qwen2.5-VL-ViT", - } - - direct = lance_reference.build_parser().parse_args( - [ - "--reference-repo", - str(checkout), - "--model", - "model", - "--image", - str(tmp_path / "image.png"), - "--prompt", - "describe", - "--max-new-tokens", - "4", - "--warmup", - "1", - "--iterations", - "2", - "--output", - str(tmp_path / "output.json"), - ] - ) - assert direct.reference_repo == checkout - def test_check_fails_fast_when_selected_reference_input_is_missing( tmp_path: Path, monkeypatch, capsys @@ -2610,7 +1906,7 @@ def test_timeseries_entries_use_current_forecast_request_schema(tmp_path: Path) assert not ({"branch_input", "field_input", "trunk_input"} & entry.case.request.keys()) timesfm = next(entry for entry in resolved if entry.model.family == "timesfm") assert timesfm.case.request["frequency"] == 2 - source = (REPO / "apps/benchmark/performance/baselines/task_reference.py").read_text( + source = (REPO / "qualification_tests/benchmark_qualification/performance/references/generic_reference.py").read_text( encoding="utf-8" ) assert '_numeric_values(request, "past_values")' in source @@ -2618,602 +1914,6 @@ def test_timeseries_entries_use_current_forecast_request_schema(tmp_path: Path) assert '_numeric_values(request, "field_input")' not in source -def test_qwen3_omni_uses_the_text_generation_contract(tmp_path: Path) -> None: - _, environment = _environment(tmp_path) - # Check the builtin text-only adapter, independently of the active owned route. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "qwen3_omni.generate"] - resolved = perf.resolve_entries(selected, environment)[0] - assert resolved.spec["operation"] == "generate" - assert resolved.spec["baseline"]["output_contract"] == "exact-text" - assert resolved.case.request["max_new_tokens"] == 16 - assert "talker_max_new_tokens" not in resolved.case.request - command = perf.baseline_command(resolved, environment, tmp_path / "reference.json") - assert command[command.index("--adapter") + 1] == "hf-qwen3-omni" - assert command[command.index("--operation") + 1] == "generate" - request = json.loads(command[command.index("--request-json") + 1]) - assert request["max_new_tokens"] == 16 - assert "talker_max_new_tokens" not in request - source = (REPO / "apps/benchmark/performance/baselines/task_reference.py").read_text( - encoding="utf-8" - ) - assert '"enable_audio_output": False' in source - assert "return_audio=False" in source - - -def test_sana_reference_reports_materialized_video_shape() -> None: - video = np.stack( - [ - np.zeros((24, 32, 3), dtype=np.uint8), - np.ones((24, 32, 3), dtype=np.uint8), - ] - ) - summary = sana_wm_reference.media_summary(video) - assert summary == { - "media_type": "video", - "media_count": 2, - "num_frames": 2, - "height": 24, - "width": 32, - "channels": 3, - } - assert perf._media_shape(summary) == (2, 24, 32, 3) - source = (REPO / "apps/benchmark/performance/baselines/task_reference.py").read_text( - encoding="utf-8" - ) - assert 'request.get("action"' in source - - -def test_sana_world_request_preserves_official_camera_controls(tmp_path: Path) -> None: - model = replace( - perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional"), - task="world_model_generation", - ) - request = perf.resolve_case( - model, tmp_path / "model.bundle", selected_task="world_model_generation", - ).request - assert request["translation_speed"] == 0.055 - assert request["rotation_speed_deg"] == 1.2 - assert request["fps"] == 16 - assert request["flow_shift"] == 9.8 - assert request["no_action_overlay"] is True - - -@pytest.mark.parametrize("route", ["builtin", "script", "hf"]) -def test_reference_testcase_name_is_only_sent_to_builtin_runner(monkeypatch, tmp_path, route): - _, environment = _environment(tmp_path) - # Exercise explicit runner fixtures, not a family's active owned replacement. - _, entries, _ = perf._load_suite_file(SUITE) - entry = perf.resolve_entries( - [value for value in entries if value["id"] == "sana_wm.generate_image"], environment, - )[0] - baseline = dict(entry.spec["baseline"]) - if route == "script": - baseline.pop("adapter") - baseline["script"] = "tests/reference.py" - monkeypatch.setattr(perf, "_family_script", lambda _: tmp_path / "reference.py") - elif route == "hf": - baseline["runner"] = "hf-transformers" - entry = replace(entry, spec={**entry.spec, "baseline": baseline}) - command = perf.baseline_command(entry, environment, tmp_path / "reference.json") - assert command[command.index("--case-name") + 1] == entry.spec["id"] - if route == "builtin": - assert command[command.index("--testcase-name") + 1] == entry.case.testcase_name - parsed = task_reference.build_parser().parse_args(command[2:]) - assert parsed.testcase_name == entry.case.testcase_name - index = command.index("--testcase-name") - direct = command[2:index] + command[index + 2:] - assert task_reference.build_parser().parse_args(direct).testcase_name is None - else: - assert "--testcase-name" not in command - - -@pytest.mark.parametrize("selection", ["selected", "missing", "duplicate", ""]) -def test_sana_reference_resolves_only_exact_testcase_before_execution( - monkeypatch, tmp_path, selection, -): - manifest = tmp_path / "manifest.json" - controls = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - chosen = {"name": "selected", **controls} - manifest.write_text(json.dumps({"video_num_frames": 321, "testcases": [ - {"name": "performance-entry", **dict.fromkeys(controls, "wrong testcase")}, - chosen, *([chosen] if selection == "duplicate" else []), - ]})) - arguments = SimpleNamespace( - manifest=manifest, testcase_name="selected" if selection == "duplicate" else selection, - case_name="performance-entry", warmup=1, iterations=2, - ) - assets = REPO / "families/sana_wm/tests/assets" - request = {"prompt": "drive forward", "image_path": str(assets / "demo_0.png"), - "action": "w-320", "num_steps": 60, "cfg_scale": 5.0, "seed": 42} - options = {"reference_repo": str(tmp_path), "model_dir": str(tmp_path / "model"), - "intrinsics": str(assets / "demo_0_intrinsics.npy")} - before = deepcopy(request), deepcopy(options), manifest.read_bytes() - - def run(command, **kwargs): - assert selection == "selected", "invalid selection must fail before reference execution" - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(controls[name]) - assert "--no_action_overlay" in command - assert command[command.index("--num_frames") + 1] == "321" - raise RuntimeError("captured selected reference command") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - if selection == "selected": - with pytest.raises(RuntimeError, match="captured selected reference command"): - task_reference._run_sana_wm(arguments, request, options) - else: - with pytest.raises(ValueError, match="exactly one testcase"): - task_reference._run_sana_wm(arguments, request, options) - assert (request, options, manifest.read_bytes()) == before - - -def test_sana_reference_options_use_resolved_testcase_and_explicit_options( - monkeypatch, tmp_path: Path, -) -> None: - _, environment = _environment(tmp_path) - # This tests the existing builtin adapter even after an owned script takes over. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] - entry = perf.resolve_entries(selected, environment)[0] - original = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - assert entry.case.testcase_name != entry.spec["id"] - testcase = next(value for value in entry.manifest["testcases"] - if value["name"] == entry.case.testcase_name) - other = {**testcase, "name": entry.spec["id"], **dict.fromkeys(original, "wrong testcase")} - entry = replace(entry, manifest={**entry.manifest, "testcases": [other, testcase]}) - semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") - request = task_reference.flatten_config(semantic.request) - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text(json.dumps(entry.manifest)) - arguments = SimpleNamespace( - manifest=manifest_path, testcase_name=entry.case.testcase_name, warmup=1, iterations=2, - ) - before = json.dumps(entry.manifest, sort_keys=True) - options = perf._adapter_options(entry, environment) - assert not original.keys() & options.keys() - assert "action" not in options and "prompt" not in options - assert options["reference_repo"] == str(Path(environment.references["sana_repo"]).resolve()) - assert options["model_dir"] == str(Path(environment.references["sana_model"]).resolve()) - options["intrinsics"] = str(entry.model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy") - expected = original - - def run(command, **kwargs): - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(expected[name]) - assert ("--no_action_overlay" in command) is expected["no_action_overlay"] - assert command[command.index("--action") + 1] == request["action"] - assert Path(command[command.index("--prompt") + 1]).read_text() == request["prompt"] - raise RuntimeError("captured reference options") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - original_options = deepcopy(options) - with pytest.raises(RuntimeError, match="captured reference options"): - task_reference._run_sana_wm(arguments, request, options) - assert options == original_options - explicit = {"translation_speed": 0.0, "rotation_speed_deg": 0.0, - "fps": 0, "flow_shift": 0.0, "no_action_overlay": False} - configured = {**entry.spec["baseline"].get("adapter_options", {}), **explicit} - entry = replace(entry, spec={**entry.spec, "baseline": {**entry.spec["baseline"], "adapter_options": configured}}) - options = perf._adapter_options(entry, environment) - assert {name: options[name] for name in original} == explicit - assert options["no_action_overlay"] is False - options["intrinsics"] = original_options["intrinsics"] - explicit_options = deepcopy(options) - expected = explicit - with pytest.raises(RuntimeError, match="captured reference options"): - task_reference._run_sana_wm(arguments, request, options) - assert options == explicit_options - assert request == task_reference.flatten_config(semantic.request) - assert json.loads(manifest_path.read_text()) == entry.manifest - assert json.dumps(entry.manifest, sort_keys=True) == before - assert entry.spec["baseline"]["adapter_options"] == configured - - -def test_sana_semantic_request_metadata_moves_only_to_reference_options( - monkeypatch, tmp_path: Path, -) -> None: - _, environment = _environment(tmp_path) - # Keep the builtin reference contract independent of canonical script selection. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] - entry = perf.resolve_entries(selected, environment)[0] - semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") - entry = replace(entry, case=semantic) - options = perf._adapter_options(entry, environment) - expected = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - assert not expected.keys() & semantic.request.keys() - assert not expected.keys() & options.keys() - command = perf.baseline_command(entry, environment, tmp_path / "reference.json") - arguments = task_reference.build_parser().parse_args(command[2:]) - - def run(command, **kwargs): - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(expected[name]) - assert "--no_action_overlay" in command - raise RuntimeError("captured reference metadata") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - with pytest.raises(RuntimeError, match="captured reference metadata"): - task_reference._run_sana_wm(arguments, task_reference.flatten_config(semantic.request), options) - assert semantic.request["action"] == entry.manifest["testcases"][0]["action"] - assert semantic.request["seed"] == 42 and semantic.request["num_steps"] == 60 - assert semantic.request["cfg_scale"] == 5.0 - - -def test_sana_reference_calls_official_pipeline_with_exact_workload( - monkeypatch, tmp_path: Path -) -> None: - captured: dict[str, object] = {"generate": []} - reference_repo = tmp_path / "Sana" - reference_repo.mkdir() - model_dir = tmp_path / "model" - (model_dir / "dit").mkdir(parents=True) - (model_dir / "refiner/text_encoder").mkdir(parents=True) - (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") - (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") - image = tmp_path / "image.png" - image.write_bytes(b"image") - prompt = tmp_path / "prompt.txt" - prompt.write_text("drive forward", encoding="utf-8") - intrinsics = tmp_path / "intrinsics.npy" - intrinsics.write_bytes(b"intrinsics") - output = tmp_path / "result.json" - - class FakeImage: - def convert(self, mode): - captured["image_mode"] = mode - return self - - def save(self, path): - Path(path).write_bytes(b"png") - return None - - pil = ModuleType("PIL") - pil.Image = SimpleNamespace(open=lambda path: FakeImage(), fromarray=lambda value: FakeImage()) - monkeypatch.setitem(sys.modules, "PIL", pil) - - synchronize_calls = [] - torch = ModuleType("torch") - torch.cuda = SimpleNamespace( - is_available=lambda: True, - synchronize=lambda: synchronize_calls.append(True), - ) - monkeypatch.setitem(sys.modules, "torch", torch) - - pyrallis = ModuleType("pyrallis") - - def parse_config(**kwargs): - captured["parse"] = kwargs - return "config" - - pyrallis.parse = parse_config - monkeypatch.setitem(sys.modules, "pyrallis", pyrallis) - - class RefinerSettings: - def __init__(self, **kwargs): - captured["refiner"] = kwargs - - class GenerationParams: - def __init__(self, **kwargs): - captured["generation"] = kwargs - - class Pipeline: - def __init__(self, **kwargs): - captured["pipeline"] = kwargs - - def generate(self, *args): - captured["generate"].append(args) - return { - "video": np.zeros((321, 24, 32, 3), dtype=np.uint8), - "c2w": "camera", - } - - trajectory = np.zeros((321, 4, 4), dtype=np.float32) - official = SimpleNamespace( - InferenceConfig=object, - RefinerSettings=RefinerSettings, - GenerationParams=GenerationParams, - SanaWMPipeline=Pipeline, - action_string_to_c2w=lambda action, **kwargs: ( - captured.update(action=(action, kwargs)) or trajectory - ), - _snap_num_frames=lambda value, **kwargs: value, - resize_and_center_crop=lambda value: ("cropped", (1, 1), (2, 2), (0, 0)), - load_intrinsics=lambda path, frames: ( - captured.update(load_intrinsics=(path, frames)) or "raw-intrinsics" - ), - transform_intrinsics_for_crop=lambda value, *sizes: ( - captured.update(transform_intrinsics=(value, sizes)) or "intrinsics" - ), - apply_overlay=lambda *_: (_ for _ in ()).throw( - AssertionError("no-action-overlay must skip overlay") - ), - ) - monkeypatch.setattr(sana_wm_reference, "_official_module", lambda path: official) - monkeypatch.setattr( - sys, - "argv", - [ - "sana_wm_reference.py", - "--reference-repo", - str(reference_repo), - "--image", - str(image), - "--model-dir", - str(model_dir), - "--prompt", - str(prompt), - "--action", - "w-320", - "--intrinsics", - str(intrinsics), - "--num_frames", - "321", - "--fps", - "16", - "--step", - "60", - "--cfg_scale", - "5.0", - "--flow_shift", - "9.8", - "--seed", - "42", - "--refiner_seed", - "42", - "--translation_speed", - "0.055", - "--rotation_speed_deg", - "1.2", - "--no_action_overlay", - "--warmup", - "1", - "--iterations", - "2", - "--output", - str(output), - ], - ) - - assert sana_wm_reference.main() == 0 - assert captured["action"] == ( - "w-320", - {"translation_speed": 0.055, "rotation_speed_deg": 1.2}, - ) - assert captured["refiner"] == { - "root": model_dir / "refiner", - "gemma_root": model_dir / "refiner/text_encoder", - "seed": 42, - } - assert captured["generation"] == { - "num_frames": 321, - "fps": 16, - "step": 60, - "cfg_scale": 5.0, - "flow_shift": 9.8, - "seed": 42, - } - assert len(captured["generate"]) == 3 - assert len(synchronize_calls) == 5 - result = json.loads(output.read_text(encoding="utf-8")) - assert result["output_summary"]["num_frames"] == 321 - - -def test_sana_reference_requires_action_from_current_request(tmp_path: Path) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json" - ) - with pytest.raises(ValueError, match="non-empty action"): - task_reference._run_sana_wm( - arguments, - {"prompt": "drive", "image_path": "assets/demo_0.png"}, - {"reference_repo": str(checkout)}, - ) - - -@pytest.mark.parametrize("frame_controls", [{"num_frames": 321}, {}]) -def test_sana_task_reference_uses_one_explicit_official_command( - monkeypatch, tmp_path: Path, frame_controls: dict, -) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - model_dir = tmp_path / "model" - model_dir.mkdir() - captured = {} - - def run(command, **kwargs): - captured["command"] = command - captured["kwargs"] = kwargs - output = Path(command[command.index("--output") + 1]) - output.write_text( - json.dumps( - { - "samples_ms": [1.0, 2.0], - "output_summary": { - "media_type": "video", - "media_count": 321, - "num_frames": 321, - "height": 704, - "width": 1280, - "channels": 3, - }, - } - ), - encoding="utf-8", - ) - return SimpleNamespace(returncode=0, stderr="") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", - warmup=1, - iterations=2, - ) - result = task_reference._run_sana_wm( - arguments, - { - "prompt": "drive forward", - "image_path": "assets/demo_0.png", - "action": "w-80,jw-40,w-40,lw-60,w-100", - "translation_speed": 0.055, - "rotation_speed_deg": 1.2, - **frame_controls, - "fps": 16, - "num_steps": 60, - "cfg_scale": 5.0, - "flow_shift": 9.8, - "seed": 42, - "no_action_overlay": True, - }, - { - "reference_repo": str(checkout), - "model_dir": str(model_dir), - "intrinsics": "assets/demo_0_intrinsics.npy", - }, - ) - - command = captured["command"] - assert command[command.index("--reference-repo") + 1] == str(checkout.resolve()) - assert command[command.index("--translation_speed") + 1] == "0.055" - assert command[command.index("--rotation_speed_deg") + 1] == "1.2" - assert command[command.index("--num_frames") + 1] == "321" - assert command[command.index("--fps") + 1] == "16" - assert command[command.index("--flow_shift") + 1] == "9.8" - assert command[command.index("--refiner_seed") + 1] == "42" - assert command[command.index("--warmup") + 1] == "1" - assert command[command.index("--iterations") + 1] == "2" - assert "--no_action_overlay" in command - assert "env" not in captured["kwargs"] - assert result[0] == [1.0, 2.0] - assert result[1]["num_frames"] == 321 - - -@pytest.mark.parametrize("controls,manifest_frames,expected", [ - ({}, 321, "321"), - ({"num_frames": 7}, 321, "7"), - ({"num_frames": 0}, 321, "0"), - ({"config": {"num_frames": 0}}, 321, "0"), - ({"num_frames": 7}, None, "7"), - ({}, None, None), -]) -def test_sana_semantic_reference_uses_manifest_frames_only_when_absent( - monkeypatch, tmp_path: Path, controls: dict, manifest_frames: int | None, expected: str | None, -) -> None: - model = perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional") - model = replace(model, testcases=({**model.testcases[0], **controls},)) - case = perf.resolve_case(model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") - request = task_reference.flatten_config(case.request) - before = dict(request) - manifest = json.loads(model.manifest_path.read_text()) - if manifest_frames is None: - del manifest["video_num_frames"] - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text(json.dumps(manifest)) - checkout = tmp_path / "Sana" - checkout.mkdir() - options = { - "reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), - "intrinsics": str(model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy"), - } - testcase = next(value for value in manifest["testcases"] if value["name"] == case.testcase_name) - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay"): - options[name] = testcase[name] - assert name not in request - - def run(command, **kwargs): - assert command[command.index("--num_frames") + 1] == expected - assert command[command.index("--action") + 1] == request["action"] - assert command[command.index("--translation_speed") + 1] == "0.055" - assert command[command.index("--rotation_speed_deg") + 1] == "1.2" - assert command[command.index("--fps") + 1] == "16" - assert command[command.index("--flow_shift") + 1] == "9.8" - assert "--no_action_overlay" in command - raise RuntimeError("captured reference command") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - arguments = SimpleNamespace(manifest=manifest_path, warmup=1, iterations=2) - if expected is None: - with pytest.raises(KeyError, match="video_num_frames"): - task_reference._run_sana_wm(arguments, request, options) - else: - with pytest.raises(RuntimeError, match="captured reference command"): - task_reference._run_sana_wm(arguments, request, options) - assert request == before - - -@pytest.mark.parametrize("source", ["request", "config", "options"]) -@pytest.mark.parametrize("controls", [ - {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True}, - {"translation_speed": 0.0, "rotation_speed_deg": 0.0, - "fps": 0, "flow_shift": 0.0, "no_action_overlay": False}, -]) -def test_sana_reference_control_presence_and_request_precedence( - monkeypatch, tmp_path: Path, source: str, controls: dict, -) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", - warmup=1, iterations=2, - ) - request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", - "action": "w-80,jw-40,w-40,lw-60,w-100", "num_frames": 321, - "num_steps": 60, "cfg_scale": 5.0, "seed": 42} - options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), - "translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - if source == "options": - options.update(controls) - elif source == "config": - request = task_reference.flatten_config({**request, "config": controls}) - else: - request.update(controls) - before = dict(request), dict(options) - - def run(command, **kwargs): - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(controls[name]) - assert ("--no_action_overlay" in command) is controls["no_action_overlay"] - assert command[command.index("--action") + 1] == request["action"] - assert command[command.index("--refiner_seed") + 1] == "42" - assert "env" not in kwargs - raise RuntimeError("captured reference controls") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - with pytest.raises(RuntimeError, match="captured reference controls"): - task_reference._run_sana_wm(arguments, request, options) - assert (request, options) == before - - -@pytest.mark.parametrize("missing", [ - "translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay", -]) -def test_sana_reference_control_missing_from_request_and_options_still_errors( - monkeypatch, tmp_path: Path, missing: str, -) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", - warmup=1, iterations=2, - ) - request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", "action": "w-320", - "num_frames": 321, "num_steps": 60, "cfg_scale": 5.0, "seed": 42} - options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), - "translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - del options[missing] - monkeypatch.setattr(task_reference.subprocess, "run", lambda *args, **kwargs: pytest.fail("missing control must fail before reference execution")) - with pytest.raises(KeyError, match=missing): - task_reference._run_sana_wm(arguments, request, options) - def test_output_contracts_are_closed_and_semantic(tmp_path: Path) -> None: _, environment = _environment(tmp_path) diff --git a/qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py b/qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py new file mode 100644 index 0000000000..c63c67c045 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path + +from qualification_tests.benchmark_qualification.performance import reference_protocol + + +def _command(tmp_path: Path, *, script: Path | None, adapter: str | None) -> list[str]: + return reference_protocol.command( + python=Path("/reference/bin/python"), + generic_runner=Path("/repo/generic_reference.py"), + script=script, + family="example", + operation="generate", + manifest=tmp_path / "model.json", + selected_task="text_generation" if script else None, + testcase_name="smoke", + adapter=adapter, + adapter_options={"option": 1}, + timing_contract={"timing_scope": "task-model-call-wall"}, + padding="longest", + model="owner/model", + revision="a" * 40, + request={"prompt": "hello"}, + precision="fp32", + mode="hf-eager", + warmup=0, + iterations=1, + case_name="example", + output=tmp_path / "reference.json", + trust_remote_code=False, + local_files_only=True, + ) + + +def test_generic_reference_command_has_no_family_dispatch(tmp_path: Path) -> None: + command = _command(tmp_path, script=None, adapter="hf-transformers-vision") + + assert command[:2] == ["/reference/bin/python", "/repo/generic_reference.py"] + assert command[command.index("--adapter") + 1] == "hf-transformers-vision" + assert "--family" not in command + assert command[command.index("--testcase-name") + 1] == "smoke" + assert json.loads(command[command.index("--request-json") + 1]) == {"prompt": "hello"} + + +def test_family_reference_command_names_only_its_owner(tmp_path: Path) -> None: + script = tmp_path / "families/example/tests/benchmark/reference.py" + command = _command(tmp_path, script=script, adapter=None) + + assert command[:2] == ["/reference/bin/python", str(script)] + assert command[command.index("--family") + 1] == "example" + assert command[command.index("--selected-task") + 1] == "text_generation" + assert "--adapter" not in command + assert "--testcase-name" not in command + + +def test_reference_command_rejects_ambiguous_implementation(tmp_path: Path) -> None: + try: + _command(tmp_path, script=None, adapter=None) + except ValueError as error: + assert "exactly one" in str(error) + else: + raise AssertionError("ambiguous reference must fail") diff --git a/qualification_tests/benchmark_qualification/performance/tests/test_reporting.py b/qualification_tests/benchmark_qualification/performance/tests/test_reporting.py new file mode 100644 index 0000000000..d4c655b620 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/tests/test_reporting.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest + +from qualification_tests.benchmark_qualification.performance import matrix as perf_matrix + + +@pytest.mark.parametrize( + "schema, row_key, reference_key", + [("v1", "cases", "baseline"), ("v2", "rows", "reference")], +) +def test_performance_report_preserves_historical_measurements( + tmp_path, schema, row_key, reference_key +): + scope = "model_call_wall" if schema == "v1" else "public_task_call_wall" + results = { + "schema_version": "trtmc.perf-matrix/" + schema, + "status": "completed", + "selected_entry_ids": ["example.generate"], + row_key: [ + { + "id": "example.generate", + "model": "example", + "operation": "generate", + "status": "green", + "candidate": {"samples_ms": [8.0, 10.0, 12.0], "timing_scope": scope}, + reference_key: { + "samples_ms": [12.0, 14.0, 16.0], + "measurement_policy": {"timing_scope": "public_operation_call_wall"}, + }, + "commands": { + "candidate": {"argv": ["trtmc-bench", "run", "--model", "example"]} + }, + } + ], + } + original = json.dumps(results) + (tmp_path / "results.json").write_text(original) + assert perf_matrix.main(["report", str(tmp_path)]) == 0 + report = json.loads((tmp_path / "report.json").read_text()) + document = (tmp_path / "report.html").read_text() + assert report["source_schema_version"] == results["schema_version"] + assert report["rows"][0][reference_key] == results[row_key][0][reference_key] + assert "p50: 10.000 ms" in document + assert "p95: 11.800 ms" in document + assert scope in document + assert "public_operation_call_wall" in document + assert "trtmc-bench" in document + assert 'id="filter"' in document + assert (tmp_path / "results.json").read_text() == original + if schema == "v1": + with pytest.raises(perf_matrix.PerfMatrixError, match="unsupported results schema"): + perf_matrix._load_results(tmp_path) diff --git a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py b/qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py similarity index 99% rename from apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py rename to qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py index d536b2ab6b..f730c71d05 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py @@ -10,7 +10,7 @@ import pytest -import tools.perf_matrix as perf +from qualification_tests.benchmark_qualification.performance import matrix as perf OPERATIONS = { diff --git a/apps/benchmark/trtmc_benchmark/tests/test_timing_contracts.py b/qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py similarity index 81% rename from apps/benchmark/trtmc_benchmark/tests/test_timing_contracts.py rename to qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py index 1bab6383d6..968b4629e3 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_timing_contracts.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py @@ -11,9 +11,9 @@ import pytest -import tools.perf_matrix as perf -from apps.benchmark.performance.baselines import task_reference -from apps.benchmark.performance.baselines.timing_contracts import timing_contract +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance.references import generic_reference +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract from trtmc_benchmark.types import ModelDescriptor @@ -23,7 +23,7 @@ def _declaration(scope="task-pipeline-call-wall", preparation=True, assets=False def test_every_release_entry_preserves_its_existing_explicit_timing(): - _, entries, _ = perf.load_suite(perf.REPOSITORY / "apps/benchmark/performance/release.yaml") + _, entries, _ = perf.load_suite(perf.REPOSITORY / "qualification_tests/benchmark_qualification/performance/config/release.yaml") assert entries for entry in entries: baseline = entry["baseline"] @@ -102,7 +102,7 @@ def invoke(): calls.append("invoke") return {"value": 7} - return task_reference.Session(invoke, "fixture", **actual) + return generic_reference.Session(invoke, "fixture", **actual) def measure(session, warmup, iterations): calls.append("measure") @@ -110,9 +110,9 @@ def measure(session, warmup, iterations): summary = session.invoke() return [2.0] * iterations, summary - monkeypatch.setitem(task_reference.LOADERS, "hf-transformers-vision", load) - monkeypatch.setattr(task_reference, "_measure", measure) - monkeypatch.setattr(task_reference, "_environment", lambda: {}) + monkeypatch.setitem(generic_reference.LOADERS, "hf-transformers-vision", load) + monkeypatch.setattr(generic_reference, "_measure", measure) + monkeypatch.setattr(generic_reference, "_environment", lambda: {}) return calls @@ -121,7 +121,7 @@ def test_real_session_metadata_not_family_name_controls_validation(tmp_path, mon actual = _declaration("task-model-call-wall", False) calls = _loader(monkeypatch, actual) arguments = _arguments(tmp_path, actual, family) - assert task_reference.run(arguments) == 0 + assert generic_reference.run(arguments) == 0 result = json.loads(arguments.output.read_text(encoding="utf-8")) assert {name: result["measurement_policy"][name] for name in actual} == actual assert calls == ["load", "measure", "invoke", "invoke", "invoke"] @@ -135,7 +135,7 @@ def test_session_timing_drift_is_rejected_before_warmup_or_measurement(tmp_path, calls = _loader(monkeypatch, actual) arguments = _arguments(tmp_path, _declaration()) with pytest.raises(RuntimeError, match="timing drifted"): - task_reference.run(arguments) + generic_reference.run(arguments) assert calls == ["load"] assert not arguments.output.exists() @@ -144,26 +144,15 @@ def test_unconfigured_fixed_loader_reports_its_actual_session_boundary(tmp_path, actual = _declaration("task-model-call-wall", False) _loader(monkeypatch, actual) arguments = _arguments(tmp_path, {}) - assert task_reference.run(arguments) == 0 + assert generic_reference.run(arguments) == 0 result = json.loads(arguments.output.read_text(encoding="utf-8")) assert {name: result[name] for name in actual} == actual def test_ambiguous_embedding_reference_requires_existing_explicit_timing_option(): with pytest.raises(ValueError, match="requires explicit --timing-contract-json"): - task_reference._load_embedding(SimpleNamespace(family="bert"), {"prompt": "hello"}, {}) + generic_reference._load_embedding(SimpleNamespace(family="bert"), {"prompt": "hello"}, {}) with pytest.raises(ValueError, match="preloads its assets"): - task_reference._load_embedding( + generic_reference._load_embedding( SimpleNamespace(timing_contract_json=json.dumps(_declaration(assets=True))), {}, {}, ) - - -def test_delegated_runner_cannot_echo_the_expected_asset_flag(tmp_path, monkeypatch): - arguments = _arguments(tmp_path, _declaration(assets=True)) - arguments.adapter, arguments.mode = "upstream-lance", "pytorch-eager" - monkeypatch.setattr(task_reference, "_run_lance", lambda *_: ( - [2.0, 3.0], {}, "fixture", "task-pipeline-call-wall", True, False, - )) - with pytest.raises(RuntimeError, match="timing drifted"): - task_reference.run(arguments) - assert not arguments.output.exists() diff --git a/qualification_tests/benchmark_qualification/performance/types.py b/qualification_tests/benchmark_qualification/performance/types.py new file mode 100644 index 0000000000..f7c3172b22 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/types.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stable values exchanged by Performance Qualification modules.""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + + +class PerfMatrixError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Environment: + name: str + trtmc_bench: Path + worker: Path + hf_runner: Path + task_runner: Path + results_root: Path + scratch_root: Path + bundle_cache: Path + bundle_roots: tuple[Path, ...] + runtime_root: Path + bundle_retention: str + local_files_only: bool + timeout_seconds: int + references: Mapping[str, str] + reference_python: Path = Path(sys.executable) + storage_root: Path | None = None + hf_cache_mode: str = "shared" + hf_cache_retention: str = "retain" + + +@dataclass(frozen=True) +class ResolvedEntry: + spec: Mapping[str, Any] + model: Any + case: Any + manifest: Mapping[str, Any] + reference_precision: str + baseline_timing: Mapping[str, Any] diff --git a/tools/benchmark_qualification/references/hf_encoder.py b/qualification_tests/benchmark_qualification/references/hf_encoder.py similarity index 100% rename from tools/benchmark_qualification/references/hf_encoder.py rename to qualification_tests/benchmark_qualification/references/hf_encoder.py diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/qualification_tests/benchmark_qualification/references/hf_text_generation.py similarity index 100% rename from tools/benchmark_qualification/references/hf_text_generation.py rename to qualification_tests/benchmark_qualification/references/hf_text_generation.py diff --git a/tools/benchmark_qualification/references/speech_io.py b/qualification_tests/benchmark_qualification/references/speech_io.py similarity index 100% rename from tools/benchmark_qualification/references/speech_io.py rename to qualification_tests/benchmark_qualification/references/speech_io.py diff --git a/tools/benchmark_qualification/references/timm_image_classification.py b/qualification_tests/benchmark_qualification/references/timm_image_classification.py similarity index 100% rename from tools/benchmark_qualification/references/timm_image_classification.py rename to qualification_tests/benchmark_qualification/references/timm_image_classification.py diff --git a/tools/benchmark_qualification/runtime.py b/qualification_tests/benchmark_qualification/runtime.py similarity index 94% rename from tools/benchmark_qualification/runtime.py rename to qualification_tests/benchmark_qualification/runtime.py index ce3bf2f903..5558643076 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/qualification_tests/benchmark_qualification/runtime.py @@ -9,7 +9,6 @@ import html import json import os -import shlex import site import shutil import subprocess @@ -160,7 +159,7 @@ def prepare_bundle( descriptor: Path, ) -> Path: command = [ - str(benchmark_executable(case, context)), + str(context.trtmc_bench), "run", "--model", str(descriptor), @@ -302,23 +301,6 @@ def reference_environment_options( return result -def benchmark_executable(case: QualificationCase, context: RuntimeContext) -> Path: - """Run user-facing benchmark and bundle build with the family Python environment.""" - python = reference_python(case, context) - digest = hashlib.sha256(f"{python.resolve()}\0{context.trtmc_bench}".encode()).hexdigest()[:12] - launcher = context.environment_root / "launchers" / f"{case.family}-{digest}" - contents = ( - f'#!/bin/sh\nexec {shlex.quote(str(python))} {shlex.quote(str(context.trtmc_bench))} "$@"\n' - ) - if not launcher.is_file() or launcher.read_text(encoding="utf-8") != contents: - launcher.parent.mkdir(parents=True, exist_ok=True) - temporary = launcher.with_name(f".{launcher.name}.{os.getpid()}.tmp") - temporary.write_text(contents, encoding="utf-8") - temporary.chmod(0o755) - os.replace(temporary, launcher) - return launcher - - def _inherit_parent_site_packages(environment: Path) -> None: child_packages = sorted(environment.glob("lib/python*/site-packages")) if len(child_packages) != 1: diff --git a/tools/model_benchmark.py b/tools/model_benchmark.py index bc2ca63097..80abf73b18 100644 --- a/tools/model_benchmark.py +++ b/tools/model_benchmark.py @@ -21,15 +21,15 @@ if str(REPOSITORY) not in sys.path: sys.path.insert(0, str(REPOSITORY)) -from tools.benchmark_qualification.accuracy import run_accuracy # noqa: E402 -from tools.benchmark_qualification.catalog import ( # noqa: E402 +from qualification_tests.benchmark_qualification.accuracy import run_accuracy # noqa: E402 +from qualification_tests.benchmark_qualification.catalog import ( # noqa: E402 QualificationCase, QualificationError, discover, select, ) -from tools.benchmark_qualification.performance import run_performance # noqa: E402 -from tools.benchmark_qualification.runtime import context_from_args, write_result # noqa: E402 +from qualification_tests.benchmark_qualification.performance.qualification import run_performance # noqa: E402 +from qualification_tests.benchmark_qualification.runtime import context_from_args, write_result # noqa: E402 def parser() -> argparse.ArgumentParser: diff --git a/tools/test_impact.py b/tools/test_impact.py index 254a628dad..090794ea1f 100644 --- a/tools/test_impact.py +++ b/tools/test_impact.py @@ -24,7 +24,7 @@ "README.md", } MODEL_PROOF_NEUTRAL_FILES = { - "apps/benchmark/performance/release.yaml", + "qualification_tests/benchmark_qualification/performance/config/release.yaml", } SHARED_PREFIXES = ( ".github/", @@ -33,6 +33,7 @@ "core/", "examples/", "plugins/", + "qualification_tests/", "requirements/", "third_party/", "tools/", diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 920a9de8b7..e807bc9fe4 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -508,63 +508,62 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/legal_headers.py", "tools/model_ci.py", "tools/model_benchmark.py", - "tools/perf_matrix.py", "tools/pr_metadata.py", "tools/test_impact.py", - "tools/benchmark_qualification/__init__.py", - "tools/benchmark_qualification/accuracy.py", - "tools/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml", - "tools/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml", - "tools/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml", - "tools/benchmark_qualification/benchmarks/embedding_performance.yaml", - "tools/benchmark_qualification/benchmarks/encoder_performance.yaml", - "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", - "tools/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml", - "tools/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", - "tools/benchmark_qualification/benchmarks/humaneval_continuation.yaml", - "tools/benchmark_qualification/benchmarks/image_classification_performance.yaml", - "tools/benchmark_qualification/benchmarks/image_features_performance.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_classification.yaml", - "tools/benchmark_qualification/benchmarks/coco2017_object_detection.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", - "tools/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", - "tools/benchmark_qualification/benchmarks/librispeech_transcription.yaml", - "tools/benchmark_qualification/benchmarks/media_generation_performance.yaml", - "tools/benchmark_qualification/benchmarks/metric_geometry_parity.yaml", - "tools/benchmark_qualification/benchmarks/metric_geometry_performance.yaml", - "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", - "tools/benchmark_qualification/benchmarks/object_detection_performance.yaml", - "tools/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml", - "tools/benchmark_qualification/benchmarks/reranking_performance.yaml", - "tools/benchmark_qualification/benchmarks/robot_action_parity.yaml", - "tools/benchmark_qualification/benchmarks/robot_action_performance.yaml", - "tools/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", - "tools/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", - "tools/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", - "tools/benchmark_qualification/benchmarks/task_output_parity.yaml", - "tools/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", - "tools/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", - "tools/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml", - "tools/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml", - "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", - "tools/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml", - "tools/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml", - "tools/benchmark_qualification/benchmarks/vision_language_performance.yaml", - "tools/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", - "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", - "tools/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", - "tools/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml", - "tools/benchmark_qualification/catalog.py", - "tools/benchmark_qualification/datasets.py", - "tools/benchmark_qualification/performance.py", - "tools/benchmark_qualification/references/hf_encoder.py", - "tools/benchmark_qualification/references/hf_text_generation.py", - "tools/benchmark_qualification/references/speech_io.py", - "tools/benchmark_qualification/references/timm_image_classification.py", - "tools/benchmark_qualification/runtime.py", + "qualification_tests/benchmark_qualification/__init__.py", + "qualification_tests/benchmark_qualification/accuracy.py", + "qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml", + "qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/encoder_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml", + "qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/image_classification_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml", + "qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml", + "qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml", + "qualification_tests/benchmark_qualification/catalog.py", + "qualification_tests/benchmark_qualification/datasets.py", + "qualification_tests/benchmark_qualification/performance/qualification.py", + "qualification_tests/benchmark_qualification/references/hf_encoder.py", + "qualification_tests/benchmark_qualification/references/hf_text_generation.py", + "qualification_tests/benchmark_qualification/references/speech_io.py", + "qualification_tests/benchmark_qualification/references/timm_image_classification.py", + "qualification_tests/benchmark_qualification/runtime.py", "tools/prepare_coco_detection_dataset.py", "tools/ci/__init__.py", "tools/ci/__main__.py", @@ -579,6 +578,10 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/ci/quality.py", "tools/ci/stage.py", } + migrated_qualification = { + path for path in expected_tools if path.startswith("qualification_tests/") + } + expected_tools -= migrated_qualification expected_tool_tests = { "tools/tests/__init__.py", "tools/tests/test_architecture.py", @@ -595,6 +598,39 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/tests/test_pr_metadata.py", "tools/tests/test_public_source_hygiene.py", } + expected_qualification = migrated_qualification | { + "qualification_tests/__init__.py", + "qualification_tests/benchmark_qualification/__init__.py", + "qualification_tests/benchmark_qualification/performance/__init__.py", + "qualification_tests/benchmark_qualification/performance/__main__.py", + "qualification_tests/benchmark_qualification/performance/README.md", + "qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml", + "qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml", + "qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml", + "qualification_tests/benchmark_qualification/performance/config/release.yaml", + "qualification_tests/benchmark_qualification/performance/config/requirements.txt", + "qualification_tests/benchmark_qualification/performance/matrix.py", + "qualification_tests/benchmark_qualification/performance/reference_harness.py", + "qualification_tests/benchmark_qualification/performance/reference_protocol.py", + "qualification_tests/benchmark_qualification/performance/reporting.py", + "qualification_tests/benchmark_qualification/performance/references/__init__.py", + "qualification_tests/benchmark_qualification/performance/references/audio_reference.py", + "qualification_tests/benchmark_qualification/performance/references/hf_transformers.py", + "qualification_tests/benchmark_qualification/performance/references/generic_reference.py", + "qualification_tests/benchmark_qualification/performance/references/timing_contracts.py", + "qualification_tests/benchmark_qualification/performance/runner.py", + "qualification_tests/benchmark_qualification/performance/stability.py", + "qualification_tests/benchmark_qualification/performance/types.py", + "qualification_tests/benchmark_qualification/performance/tests/__init__.py", + "qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py", + "qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py", + "qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py", + "qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py", + "qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py", + "qualification_tests/benchmark_qualification/performance/tests/test_reporting.py", + "qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py", + "qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py", + } expected_cmake = {"cmake/trtmcConfig.cmake.in"} expected_third_party = { "third_party/stb/stb_image.h", @@ -616,6 +652,7 @@ def files(root: str) -> set[str]: tool_tests = {path for path in tool_files if path.startswith("tools/tests/")} assert tool_files - tool_tests == expected_tools assert tool_tests == expected_tool_tests + assert files("qualification_tests") == expected_qualification assert files("cmake") == expected_cmake assert files("third_party") == expected_third_party @@ -667,7 +704,7 @@ def test_applications_depend_only_on_public_model_connect_surfaces() -> None: REPO / "examples", ) application_files = [path for root in application_roots for path in root.rglob("*")] + [ - REPO / "tools/perf_matrix.py" + REPO / "qualification_tests/benchmark_qualification/performance/matrix.py" ] for path in application_files: if not path.is_file() or "__pycache__" in path.parts: diff --git a/tools/tests/test_e2e_evidence.py b/tools/tests/test_e2e_evidence.py index 2cc20a575d..917af49ead 100644 --- a/tools/tests/test_e2e_evidence.py +++ b/tools/tests/test_e2e_evidence.py @@ -1554,7 +1554,7 @@ def test_demo_shared_chart_keeps_scope_and_rejects_invalid_values() -> None: "ocr", ], ) -def test_text_task_references_are_visible_without_opening_details( +def test_text_generic_references_are_visible_without_opening_details( tmp_path: Path, task: str ) -> None: data = _case() diff --git a/tools/tests/test_family_impact.py b/tools/tests/test_family_impact.py index 2f6b395527..5cb0ac3279 100644 --- a/tools/tests/test_family_impact.py +++ b/tools/tests/test_family_impact.py @@ -49,7 +49,7 @@ def test_release_performance_policy_does_not_expand_family_scope(tmp_path: Path) impact = test_impact.classify( repo, [ - "apps/benchmark/performance/release.yaml", + "qualification_tests/benchmark_qualification/performance/config/release.yaml", "families/alpha/model.py", ], ) diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index f5d48edaec..6da332fe2d 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -15,22 +15,23 @@ import pytest from tools import model_benchmark, prepare_coco_detection_dataset -from apps.benchmark.performance.baselines import hf_transformers, task_reference -from apps.benchmark.performance.baselines.timing_contracts import timing_contract -from tools.benchmark_qualification import accuracy as qualification_accuracy -from tools.benchmark_qualification import performance as qualification_performance -from tools.benchmark_qualification import runtime as qualification_runtime -from tools.benchmark_qualification.catalog import ( +from qualification_tests.benchmark_qualification.performance.references import hf_transformers, generic_reference +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract +from qualification_tests.benchmark_qualification import accuracy as qualification_accuracy +from qualification_tests.benchmark_qualification.performance import ( + qualification as qualification_performance, +) +from qualification_tests.benchmark_qualification import runtime as qualification_runtime +from qualification_tests.benchmark_qualification.catalog import ( QualificationCase, QualificationError, discover, load_benchmark, ) -from tools.benchmark_qualification.datasets import Dataset, resolve_dataset -from tools.benchmark_qualification.references import hf_encoder, hf_text_generation -from tools.benchmark_qualification.runtime import ( +from qualification_tests.benchmark_qualification.datasets import Dataset, resolve_dataset +from qualification_tests.benchmark_qualification.references import hf_encoder, hf_text_generation +from qualification_tests.benchmark_qualification.runtime import ( RuntimeContext, - benchmark_executable, prepare_bundle, reference_environment_options, reference_python, @@ -1320,14 +1321,19 @@ def reference(command, *_args, **_kwargs): assert result["metrics"]["candidate_knn_top1_accuracy"] == 1.0 -def test_timm_classification_uses_a_generic_task_reference_adapter() -> None: - assert "timm-classification" in task_reference.ADAPTERS - assert task_reference.LOADERS["timm-classification"] is task_reference._load_vision +def test_timm_classification_uses_a_generic_reference_adapter() -> None: + assert "timm-classification" in generic_reference.ADAPTERS + assert generic_reference.LOADERS["timm-classification"] is generic_reference._load_vision -def test_chat_asr_uses_the_generic_asr_reference_loader() -> None: - assert "hf-chat-asr" in task_reference.ADAPTERS - assert task_reference.LOADERS["hf-chat-asr"] is task_reference._load_asr +def test_glm_asr_performance_uses_its_family_reference() -> None: + case = next( + value + for value in discover(REPOSITORY) + if value.model == "glmasr-nano-fp16" and value.kind == "performance" + ) + assert case.values["reference"]["script"] == "tests/benchmark/reference.py" + assert "hf-chat-asr" not in generic_reference.ADAPTERS def test_speech_accuracy_reports_reference_and_labeled_wer(tmp_path: Path, monkeypatch) -> None: @@ -1760,15 +1766,8 @@ def test_performance_distinguishes_model_contract_failures_from_execution_errors }, } - def run_matrix(command, *_args, **_kwargs): - run_directory = context.case_artifacts(case) / "matrix/run" - run_directory.mkdir(parents=True) - matrix_row = {"id": "qualification.example.generate", **row} - (run_directory / "results.json").write_text( - json.dumps({"status": "failed", "rows": [matrix_row]}), - encoding="utf-8", - ) - return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + def run_single_case(*_args, **_kwargs): + return {"id": "qualification.example.generate", **row} monkeypatch.setattr(qualification_performance, "load_benchmark", lambda *_: definition) monkeypatch.setattr( @@ -1779,7 +1778,7 @@ def run_matrix(command, *_args, **_kwargs): monkeypatch.setattr( qualification_performance, "reference_python", lambda *_: Path(sys.executable) ) - monkeypatch.setattr(qualification_performance, "run_command", run_matrix) + monkeypatch.setattr(qualification_performance, "run_case", run_single_case) result = qualification_performance.run_performance(case, context) @@ -2108,12 +2107,7 @@ def test_shared_performance_definitions_own_complete_reference_timing() -> None: def test_internal_automation_is_separate_from_the_installed_benchmark() -> None: old_application = REPOSITORY / "apps/benchmark/qualification" assert list(old_application.rglob("*.py")) == [] - internal_sources = [ - path.read_text(encoding="utf-8") - for path in (REPOSITORY / "tools/benchmark_qualification").rglob("*.py") - ] - assert not any("ManifestCatalog" in source for source in internal_sources) - assert not any("tests/manifests" in source for source in internal_sources) + assert list((REPOSITORY / "tools/benchmark_qualification").rglob("*.py")) == [] user_sources = [ path.read_text(encoding="utf-8") for path in (REPOSITORY / "apps/benchmark").rglob("*.py") @@ -2224,7 +2218,7 @@ def test_candidate_descriptor_uses_family_environment_model_directory( verbose=False, ) monkeypatch.setattr( - "tools.benchmark_qualification.runtime.reference_python", lambda *_args: python + "qualification_tests.benchmark_qualification.runtime.reference_python", lambda *_args: python ) descriptor = write_model_descriptor( @@ -2274,12 +2268,6 @@ def test_accuracy_rejects_candidate_artifacts_outside_the_cell_directory( "prepare_bundle", lambda *_args, **_kwargs: tmp_path / "model.bundle", ) - monkeypatch.setattr( - qualification_accuracy, - "benchmark_executable", - lambda *_args, **_kwargs: tmp_path / "trtmc-bench", - ) - def candidate(command, *_args, **_kwargs): candidate_output = Path(command[command.index("--output") + 1]) candidate_output.mkdir(parents=True) @@ -2444,7 +2432,7 @@ def complete(command, *_args, **_kwargs): child_packages.mkdir(parents=True) return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - monkeypatch.setattr("tools.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("qualification_tests.benchmark_qualification.runtime.run_command", complete) monkeypatch.setattr("site.getsitepackages", lambda: [str(parent_packages)]) python = reference_python(case, context) @@ -2517,7 +2505,7 @@ def complete(command, *_args, **_kwargs): child.mkdir(parents=True) return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - monkeypatch.setattr("tools.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("qualification_tests.benchmark_qualification.runtime.run_command", complete) monkeypatch.setattr("site.getsitepackages", lambda: [str(parent_packages)]) python = reference_python(case, context) @@ -2525,57 +2513,6 @@ def complete(command, *_args, **_kwargs): assert commands[-1] == [str(python), str(hook)] -def test_benchmark_launcher_uses_the_family_python(tmp_path: Path, monkeypatch) -> None: - python = tmp_path / "family env/bin/python" - python.parent.mkdir(parents=True) - python.write_text("", encoding="utf-8") - bench = tmp_path / "user tools/trtmc-bench" - bench.parent.mkdir(parents=True) - bench.write_text("", encoding="utf-8") - case = QualificationCase( - kind="performance", - model="example", - family="example", - name="generate", - benchmark="text_generation_performance", - candidate={ - "family": "example", - "checkpoint": "example/model", - "task": "text_generation", - "precision": "fp16", - "build": {}, - }, - values={}, - source=tmp_path / "example.yaml", - reference_requirements=None, - ) - context = RuntimeContext( - repository=REPOSITORY, - artifacts=tmp_path / "artifacts", - data_root=tmp_path / "data", - environment_root=tmp_path / "envs", - bundle_cache=tmp_path / "bundles", - bundle_roots=(), - runtime_root=None, - trtmc_bench=bench, - worker=None, - datasets={}, - reference_pythons={}, - no_build=False, - verbose=False, - ) - monkeypatch.setattr( - "tools.benchmark_qualification.runtime.reference_python", lambda *_args: python - ) - - launcher = benchmark_executable(case, context) - - assert os.access(launcher, os.X_OK) - assert launcher.read_text(encoding="utf-8") == ( - f"#!/bin/sh\nexec '{python}' '{bench}' \"$@\"\n" - ) - - def test_bundle_preparation_uses_the_selected_runtime(tmp_path: Path, monkeypatch) -> None: case = QualificationCase( kind="accuracy", @@ -2627,9 +2564,10 @@ def complete(command, *_args, **_kwargs): stderr="", ) - monkeypatch.setattr("tools.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("qualification_tests.benchmark_qualification.runtime.run_command", complete) assert prepare_bundle(case, context, tmp_path, descriptor) == bundle + assert captured[0] == str(context.trtmc_bench) assert captured[captured.index("--runtime-root") + 1] == str(runtime_root) diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index 47de6f7147..a8fa291693 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -31,7 +31,7 @@ from tools.ci.pipeline import CiPipeline from tools.ci.process import CiError from tools.ci.quality import SourceQualityChecks, UnitTestRunner -from tools import perf_matrix +from qualification_tests.benchmark_qualification.performance import matrix as perf_matrix class RecordingContext: diff --git a/website/docs/reference/benchmarking.md b/website/docs/reference/benchmarking.md index a90277071b..521ab6fde6 100644 --- a/website/docs/reference/benchmarking.md +++ b/website/docs/reference/benchmarking.md @@ -9,12 +9,10 @@ does not register model families or import family implementation code. ## Single-model benchmark -Install the optional benchmark dependencies, then select an installed runtime -root containing the core runtime, TensorRT backend, and required family DSOs: +Install TRTMC, then select an installed runtime root containing the core +runtime, TensorRT backend, and required family DSOs: ```bash -python -m pip install -r apps/benchmark/performance/requirements.txt - trtmc-bench list models trtmc-bench run \ --model distilgpt2 \ @@ -87,7 +85,10 @@ reference within five percent of candidate p50 is considered equivalent. Candidate or reference execution failures are operational failures, not slow performance results. -## Release performance matrix +## Internal release performance matrix + +This repository-only workflow is CI qualification. It is separate from the +installed `trtmc-bench` user interface above and is not included in the wheel. The matrix coordinates candidate and reference runs without introducing a second model registry: @@ -97,19 +98,19 @@ export TRTMC_PERF_WORKER=/opt/trtmc/bin/trtmc_benchmark_worker export TRTMC_PERF_RUNTIME_ROOT=/opt/trtmc/lib export TRTMC_PERF_BUNDLE_CACHE=/data/trtmc-bundles -python3 tools/perf_matrix.py check \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml +python3 -m qualification_tests.benchmark_qualification.performance check \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml -python3 tools/perf_matrix.py prepare \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ +python3 -m qualification_tests.benchmark_qualification.performance prepare \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ --entry gpt2.generate \ --output artifacts/perf/bundle-preparation.json -python3 tools/perf_matrix.py run \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ +python3 -m qualification_tests.benchmark_qualification.performance run \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ --entry gpt2.generate ``` @@ -117,8 +118,8 @@ Preparation is deliberately separate and untimed. Resume or regenerate a report from stored observations with: ```bash -python3 tools/perf_matrix.py resume artifacts/perf/ -python3 tools/perf_matrix.py report artifacts/perf/ +python3 -m qualification_tests.benchmark_qualification.performance resume artifacts/perf/ +python3 -m qualification_tests.benchmark_qualification.performance report artifacts/perf/ ``` The release YAML owns model, testcase, operation, measurement, reference, and diff --git a/website/docs/tutorials/advanced/validation-and-benchmarking.md b/website/docs/tutorials/advanced/validation-and-benchmarking.md index da01dd405a..8f42503f6b 100644 --- a/website/docs/tutorials/advanced/validation-and-benchmarking.md +++ b/website/docs/tutorials/advanced/validation-and-benchmarking.md @@ -51,13 +51,13 @@ trtmc-bench run \ For release comparisons, validate and run the checked-in matrix: ```bash -python3 tools/perf_matrix.py check \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml +python3 -m qualification_tests.benchmark_qualification.performance check \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml -python3 tools/perf_matrix.py run \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ +python3 -m qualification_tests.benchmark_qualification.performance run \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ --entry gpt2.generate ``` From 1285ba8c86014629b67479f77eac40139ba14d87 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 11:47:56 +0000 Subject: [PATCH 098/100] fix(ci): satisfy source quality checks Add the required SPDX header to the qualification references package and apply the repository clang-format version to the benchmark worker changes already present in this PR. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 40 ++++++++++++++----- .../performance/references/__init__.py | 4 ++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index cc26a83683..4b6032773f 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -1168,8 +1168,9 @@ Json run_transcribe(const trtmc::Model& model, const Json& request, const Timing }); }; if (primary == trtmc::SpeechTranscription::kTask) - return run(task_for_operation(model, task_id), - [&](auto audio) { return trtmc::SpeechTranscriptionRequest{audio, language}; }); + return run(task_for_operation(model, task_id), [&](auto audio) { + return trtmc::SpeechTranscriptionRequest{audio, language}; + }); return run(task_for_operation(model, task_id), [&](auto audio) { return trtmc::SpeechTranslationRequest{audio, target, language}; }); @@ -1219,14 +1220,20 @@ Json run_generate_audio(const trtmc::Model& model, const Json& request, const Ti const auto task = task_for_operation(model, task_id); const auto config = sdk_config(request, task.config_fields(), {"prompt", "streaming", "_artifact_prefix"}); - return measure(timing, [&]() { return task.run({prompt}, config); }, observe); + return measure( + timing, [&]() { return task.run({prompt}, config); }, observe); } const auto language = language_input(request, "language"); if (primary == trtmc::TextToSpeech::kTask) { const auto task = task_for_operation(model, task_id); const auto config = sdk_config(request, task.config_fields(), {"prompt", "language", "streaming", "_artifact_prefix"}); - return measure(timing, [&]() { return task.run({prompt, language}, config); }, observe); + return measure( + timing, + [&]() { + return task.run({prompt, language}, config); + }, + observe); } if (!streaming) throw std::invalid_argument("generate_audio requires an audio generation Task"); @@ -1903,7 +1910,10 @@ Json run_embed(const trtmc::Model& model, const Json& request, const Timing& tim throw std::invalid_argument("invalid embedding role"); } return measure( - timing, [&]() { return task.run({prompt, role}, config); }, + timing, + [&]() { + return task.run({prompt, role}, config); + }, [](const trtmc::SemanticEmbeddingResult& result) { return Json{{"embedding_vectors", 1}, {"embedding_elements", result.values().size()}, @@ -1924,7 +1934,10 @@ Json run_rerank(const trtmc::Model& model, const Json& request, const Timing& ti const auto query = request.at("query").get(); const auto documents = request.at("documents").get>(); return measure( - timing, [&]() { return task.run({query, documents}, config); }, + timing, + [&]() { + return task.run({query, documents}, config); + }, [&](const trtmc::DocumentRelevanceResult& result) { return Json{{"documents", documents.size()}, {"scores", json_values(result.scores())}, @@ -3233,7 +3246,8 @@ Json run_translate(const trtmc::Model& model, const Json& request, const Timing& language_input(request, "source_language")}; const auto config = sdk_config(request, task.config_fields(), {"source_text", "source_language", "target_language"}); - return measure(timing, [&]() { return task.run(input, config); }, text_observation); + return measure( + timing, [&]() { return task.run(input, config); }, text_observation); } std::vector optional_float32(const Json& request, const char* name) { @@ -3450,7 +3464,8 @@ Json run_generate(const trtmc::Model& model, const Json& request, const Timing& auto run = [&](const auto& task, const auto& input) { const auto config = sdk_config(request, task.config_fields(), {"prompt", "token_ids", "batch_size"}); - return measure(timing, [&]() { return task.run(input, config); }, text_observation); + return measure( + timing, [&]() { return task.run(input, config); }, text_observation); }; if (primary == trtmc::ConditionalTextGeneration::kTask) return run(task_for_operation(model, task_id), @@ -3465,7 +3480,8 @@ Json run_generate(const trtmc::Model& model, const Json& request, const Timing& throw std::invalid_argument("unconditional text generation has no prompt input"); const auto task = task_for_operation(model, task_id); const auto config = sdk_config(request, task.config_fields(), {}); - return measure(timing, [&]() { return task.run(config); }, text_observation); + return measure( + timing, [&]() { return task.run(config); }, text_observation); } if (primary == trtmc::LatentConditionedTextGeneration::kTask || primary == trtmc::LatentReplayToText::kTask) @@ -3496,7 +3512,8 @@ Json run_generate(const trtmc::Model& model, const Json& request, const Timing& } auto run = [&](const auto& task, const auto& input) { const auto config = sdk_config(request, task.config_fields(), {"prompt"}); - return measure(timing, [&]() { return task.run(input, config); }, text_observation); + return measure( + timing, [&]() { return task.run(input, config); }, text_observation); }; if (primary == trtmc::CorruptedTextReconstruction::kTask) return run(task_for_operation(model, task_id), @@ -3717,7 +3734,8 @@ Json run_solve(const trtmc::Model& model, const Json& request, const Timing& tim auto run = [&](const auto& task, const auto& input, auto observe) { const auto config = sdk_config(request, task.config_fields(), {"past_values", "observed_mask", "shape"}); - return measure(timing, [&]() { return task.run(input, config); }, observe); + return measure( + timing, [&]() { return task.run(input, config); }, observe); }; if (primary == trtmc::SeriesToPointForecast::kTask) return run(task_for_operation(model, task_id), diff --git a/qualification_tests/benchmark_qualification/performance/references/__init__.py b/qualification_tests/benchmark_qualification/performance/references/__init__.py index e69de29bb2..e25025f300 100644 --- a/qualification_tests/benchmark_qualification/performance/references/__init__.py +++ b/qualification_tests/benchmark_qualification/performance/references/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reference implementations used only by internal qualification.""" From 962335c016056f8e459585d0aaccc8e501abe862 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 11:58:37 +0000 Subject: [PATCH 099/100] fix(ci): match pinned clang format Format the benchmark worker with clang-format 22.1.8, matching the public Source quality job rather than the host clang-format 14 binary. Signed-off-by: chaofengw --- apps/benchmark/native/benchmark_worker.cpp | 40 ++++++---------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 4b6032773f..cc26a83683 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -1168,9 +1168,8 @@ Json run_transcribe(const trtmc::Model& model, const Json& request, const Timing }); }; if (primary == trtmc::SpeechTranscription::kTask) - return run(task_for_operation(model, task_id), [&](auto audio) { - return trtmc::SpeechTranscriptionRequest{audio, language}; - }); + return run(task_for_operation(model, task_id), + [&](auto audio) { return trtmc::SpeechTranscriptionRequest{audio, language}; }); return run(task_for_operation(model, task_id), [&](auto audio) { return trtmc::SpeechTranslationRequest{audio, target, language}; }); @@ -1220,20 +1219,14 @@ Json run_generate_audio(const trtmc::Model& model, const Json& request, const Ti const auto task = task_for_operation(model, task_id); const auto config = sdk_config(request, task.config_fields(), {"prompt", "streaming", "_artifact_prefix"}); - return measure( - timing, [&]() { return task.run({prompt}, config); }, observe); + return measure(timing, [&]() { return task.run({prompt}, config); }, observe); } const auto language = language_input(request, "language"); if (primary == trtmc::TextToSpeech::kTask) { const auto task = task_for_operation(model, task_id); const auto config = sdk_config(request, task.config_fields(), {"prompt", "language", "streaming", "_artifact_prefix"}); - return measure( - timing, - [&]() { - return task.run({prompt, language}, config); - }, - observe); + return measure(timing, [&]() { return task.run({prompt, language}, config); }, observe); } if (!streaming) throw std::invalid_argument("generate_audio requires an audio generation Task"); @@ -1910,10 +1903,7 @@ Json run_embed(const trtmc::Model& model, const Json& request, const Timing& tim throw std::invalid_argument("invalid embedding role"); } return measure( - timing, - [&]() { - return task.run({prompt, role}, config); - }, + timing, [&]() { return task.run({prompt, role}, config); }, [](const trtmc::SemanticEmbeddingResult& result) { return Json{{"embedding_vectors", 1}, {"embedding_elements", result.values().size()}, @@ -1934,10 +1924,7 @@ Json run_rerank(const trtmc::Model& model, const Json& request, const Timing& ti const auto query = request.at("query").get(); const auto documents = request.at("documents").get>(); return measure( - timing, - [&]() { - return task.run({query, documents}, config); - }, + timing, [&]() { return task.run({query, documents}, config); }, [&](const trtmc::DocumentRelevanceResult& result) { return Json{{"documents", documents.size()}, {"scores", json_values(result.scores())}, @@ -3246,8 +3233,7 @@ Json run_translate(const trtmc::Model& model, const Json& request, const Timing& language_input(request, "source_language")}; const auto config = sdk_config(request, task.config_fields(), {"source_text", "source_language", "target_language"}); - return measure( - timing, [&]() { return task.run(input, config); }, text_observation); + return measure(timing, [&]() { return task.run(input, config); }, text_observation); } std::vector optional_float32(const Json& request, const char* name) { @@ -3464,8 +3450,7 @@ Json run_generate(const trtmc::Model& model, const Json& request, const Timing& auto run = [&](const auto& task, const auto& input) { const auto config = sdk_config(request, task.config_fields(), {"prompt", "token_ids", "batch_size"}); - return measure( - timing, [&]() { return task.run(input, config); }, text_observation); + return measure(timing, [&]() { return task.run(input, config); }, text_observation); }; if (primary == trtmc::ConditionalTextGeneration::kTask) return run(task_for_operation(model, task_id), @@ -3480,8 +3465,7 @@ Json run_generate(const trtmc::Model& model, const Json& request, const Timing& throw std::invalid_argument("unconditional text generation has no prompt input"); const auto task = task_for_operation(model, task_id); const auto config = sdk_config(request, task.config_fields(), {}); - return measure( - timing, [&]() { return task.run(config); }, text_observation); + return measure(timing, [&]() { return task.run(config); }, text_observation); } if (primary == trtmc::LatentConditionedTextGeneration::kTask || primary == trtmc::LatentReplayToText::kTask) @@ -3512,8 +3496,7 @@ Json run_generate(const trtmc::Model& model, const Json& request, const Timing& } auto run = [&](const auto& task, const auto& input) { const auto config = sdk_config(request, task.config_fields(), {"prompt"}); - return measure( - timing, [&]() { return task.run(input, config); }, text_observation); + return measure(timing, [&]() { return task.run(input, config); }, text_observation); }; if (primary == trtmc::CorruptedTextReconstruction::kTask) return run(task_for_operation(model, task_id), @@ -3734,8 +3717,7 @@ Json run_solve(const trtmc::Model& model, const Json& request, const Timing& tim auto run = [&](const auto& task, const auto& input, auto observe) { const auto config = sdk_config(request, task.config_fields(), {"past_values", "observed_mask", "shape"}); - return measure( - timing, [&]() { return task.run(input, config); }, observe); + return measure(timing, [&]() { return task.run(input, config); }, observe); }; if (primary == trtmc::SeriesToPointForecast::kTask) return run(task_for_operation(model, task_id), From 94b375e79d6c8fb1ab3e97206f2b464c67e1fe2e Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 22 Sep 2026 13:50:36 +0000 Subject: [PATCH 100/100] fix(sana-wm): load reference encoder locally The isolated CI cache stores the pinned stage-1 encoder under its immutable commit ref, while the official SANA loader resolves the default Hub ref. This makes the offline reference fail even though the exact snapshot is present. Route both E2E and qualification references through a family-owned loader that validates and opens the prepared local snapshot without Hub resolution while preserving the upstream Gemma loading semantics. Signed-off-by: chaofengw --- .../tests/benchmark/upstream_reference.py | 11 ++ families/sana_wm/tests/official_reference.py | 101 ++++++++++++++++++ .../sana_wm/tests/test_benchmark_reference.py | 11 ++ families/sana_wm/tests/test_e2e.py | 17 ++- .../sana_wm/tests/test_reference_contract.py | 78 +++++++++++++- 5 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 families/sana_wm/tests/official_reference.py diff --git a/families/sana_wm/tests/benchmark/upstream_reference.py b/families/sana_wm/tests/benchmark/upstream_reference.py index a11f89a27e..6d9f446bb8 100755 --- a/families/sana_wm/tests/benchmark/upstream_reference.py +++ b/families/sana_wm/tests/benchmark/upstream_reference.py @@ -14,6 +14,15 @@ from typing import Any +REPOSITORY = Path(__file__).resolve().parents[4] +if str(REPOSITORY) not in sys.path: + sys.path.insert(0, str(REPOSITORY)) + +from families.sana_wm.tests.official_reference import ( # noqa: E402 + install_local_stage1_text_encoder, +) + + def parser() -> argparse.ArgumentParser: value = argparse.ArgumentParser() value.add_argument("--reference-repo", required=True, type=Path) @@ -102,6 +111,7 @@ def main() -> int: required_model_paths = { "config": model_dir / "config.yaml", "model": model_dir / "dit/sana_wm_1600m_720p.safetensors", + "stage1 text encoder": model_dir / "stage1_text_encoder", "refiner": model_dir / "refiner", "refiner text encoder": model_dir / "refiner/text_encoder", } @@ -146,6 +156,7 @@ def main() -> int: gemma_root=required_model_paths["refiner text encoder"], seed=arguments.refiner_seed, ) + install_local_stage1_text_encoder(official, required_model_paths["stage1 text encoder"]) pipeline = official.SanaWMPipeline( config=config, model_path=required_model_paths["model"], diff --git a/families/sana_wm/tests/official_reference.py b/families/sana_wm/tests/official_reference.py new file mode 100644 index 0000000000..4deee8437e --- /dev/null +++ b/families/sana_wm/tests/official_reference.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Launch the pinned official SANA-WM reference with local stage-1 assets.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys +from typing import Any, Sequence + + +_STAGE1_TEXT_ENCODER_NAME = "gemma-2-2b-it" + + +def _require_stage1_text_encoder(path: Path) -> Path: + resolved = path.resolve() + required = (resolved / "config.json", resolved / "tokenizer.json") + missing = [str(value) for value in required if not value.is_file()] + if not any(resolved.glob("model*.safetensors*")): + missing.append(str(resolved / "model*.safetensors*")) + if missing: + raise FileNotFoundError( + "SANA-WM local stage-1 text encoder is incomplete: " + ", ".join(missing) + ) + return resolved + + +def load_local_stage1_text_encoder(path: Path, device: str) -> tuple[Any, Any]: + """Load the exact prepared stage-1 tokenizer and decoder without Hub resolution.""" + + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + + resolved = _require_stage1_text_encoder(path) + tokenizer = AutoTokenizer.from_pretrained(str(resolved), local_files_only=True) + tokenizer.padding_side = "right" + decoder = ( + AutoModelForCausalLM.from_pretrained( + str(resolved), + local_files_only=True, + torch_dtype=torch.bfloat16, + ) + .get_decoder() + .to(device) + ) + return tokenizer, decoder + + +def install_local_stage1_text_encoder(official: Any, path: Path) -> None: + """Replace only the official pipeline's stage-1 asset locator.""" + + resolved = _require_stage1_text_encoder(path) + + def load(name: str = "T5", device: str = "cuda") -> tuple[Any, Any]: + if name != _STAGE1_TEXT_ENCODER_NAME: + raise ValueError( + "SANA-WM expected stage-1 text encoder " + f"{_STAGE1_TEXT_ENCODER_NAME!r}, found {name!r}" + ) + return load_local_stage1_text_encoder(resolved, device) + + official.get_tokenizer_and_text_encoder = load + + +def _official_module(reference_repository: Path) -> tuple[Path, Any]: + repository = reference_repository.resolve() + entrypoint = repository / "inference_video_scripts/wm/inference_sana_wm.py" + if not entrypoint.is_file(): + raise FileNotFoundError(f"official SANA-WM entrypoint does not exist: {entrypoint}") + if str(repository) not in sys.path: + sys.path.insert(0, str(repository)) + from inference_video_scripts.wm import inference_sana_wm + + return entrypoint, inference_sana_wm + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(add_help=False) + value.add_argument("--reference-repo", required=True, type=Path) + value.add_argument("--stage1-text-encoder", required=True, type=Path) + return value + + +def main(argv: Sequence[str] | None = None) -> int: + arguments, official_arguments = parser().parse_known_args(argv) + entrypoint, official = _official_module(arguments.reference_repo) + install_local_stage1_text_encoder(official, arguments.stage1_text_encoder) + previous = sys.argv + try: + sys.argv = [str(entrypoint), *official_arguments] + result = official.main() + finally: + sys.argv = previous + return int(result or 0) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sana_wm/tests/test_benchmark_reference.py b/families/sana_wm/tests/test_benchmark_reference.py index c78dffef59..37db469b73 100644 --- a/families/sana_wm/tests/test_benchmark_reference.py +++ b/families/sana_wm/tests/test_benchmark_reference.py @@ -294,6 +294,10 @@ def test_sana_reference_calls_official_pipeline_with_exact_workload( model_dir = tmp_path / "model" (model_dir / "dit").mkdir(parents=True) (model_dir / "refiner/text_encoder").mkdir(parents=True) + stage1_text_encoder = model_dir / "stage1_text_encoder" + stage1_text_encoder.mkdir() + for name in ("config.json", "tokenizer.json", "model.safetensors"): + (stage1_text_encoder / name).write_text("{}\n", encoding="utf-8") (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") image = tmp_path / "image.png" @@ -375,6 +379,12 @@ def generate(self, *args): ), ) monkeypatch.setattr(sana_wm_reference, "_official_module", lambda path: official) + installed = {} + monkeypatch.setattr( + sana_wm_reference, + "install_local_stage1_text_encoder", + lambda module, path: installed.update(module=module, path=path), + ) monkeypatch.setattr( sys, "argv", @@ -430,6 +440,7 @@ def generate(self, *args): "gemma_root": model_dir / "refiner/text_encoder", "seed": 42, } + assert installed == {"module": official, "path": stage1_text_encoder.resolve()} assert captured["generation"] == { "num_frames": 321, "fps": 16, diff --git a/families/sana_wm/tests/test_e2e.py b/families/sana_wm/tests/test_e2e.py index 79a9d2ef77..66f8f2fe37 100644 --- a/families/sana_wm/tests/test_e2e.py +++ b/families/sana_wm/tests/test_e2e.py @@ -333,12 +333,18 @@ def _official_reference(model_dir: Path, manifest: dict, case: dict, tmp_path: P model_dir = model_dir.resolve() entrypoint = source_root / "inference_video_scripts/wm/inference_sana_wm.py" assert entrypoint.is_file(), f"declared Sana reference entrypoint is missing: {entrypoint}" + launcher = TEST_ROOT / "official_reference.py" + assert launcher.is_file(), f"Sana family reference launcher is missing: {launcher}" config = model_dir / "config.yaml" model_path = model_dir / "dit/sana_wm_1600m_720p.safetensors" + stage1_text_encoder = model_dir / "stage1_text_encoder" refiner_root = model_dir / "refiner" refiner_gemma_root = refiner_root / "text_encoder" assert config.is_file(), f"Sana reference config is missing: {config}" assert model_path.is_file(), f"Sana reference weights are missing: {model_path}" + assert stage1_text_encoder.is_dir(), ( + f"Sana reference stage-1 text encoder is missing: {stage1_text_encoder}" + ) assert refiner_root.is_dir(), f"Sana reference refiner is missing: {refiner_root}" assert refiner_gemma_root.is_dir(), ( f"Sana reference refiner text encoder is missing: {refiner_gemma_root}" @@ -349,7 +355,11 @@ def _official_reference(model_dir: Path, manifest: dict, case: dict, tmp_path: P environment["PYTHONPATH"] = str(source_root) command = [ sys.executable, - str(entrypoint), + str(launcher), + "--reference-repo", + str(source_root), + "--stage1-text-encoder", + str(stage1_text_encoder), "--image", str(_asset(case["test_image"], report_key="image")), "--prompt", @@ -544,13 +554,14 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("checkpoint", {"model_dir": str(model_dir), "hf_id": manifest.get("hf_id"), "hf_revision": manifest.get("hf_revision")}) binary, runtime_root = _runtime(manifest) bundle = tmp_path / manifest["bundle"] + prepared_model_dir = _prepared_model_dir(model_dir, manifest, tmp_path) with evidence_stage("build"): - _build(_prepared_model_dir(model_dir, manifest, tmp_path), bundle, manifest) + _build(prepared_model_dir, bundle, manifest) with evidence_stage("native"): actual = _native(binary, runtime_root, bundle, model_dir, manifest, case, tmp_path) record_evidence("native", native_snapshot(actual)) with evidence_stage("reference"): - expected = _official_reference(model_dir, manifest, case, tmp_path) + expected = _official_reference(prepared_model_dir, manifest, case, tmp_path) record_report_views(actual, expected, tmp_path / "paired-report-views") record_evidence("reference", reference_snapshot(expected)) with evidence_stage("compare"): diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index 5b0ea99d55..761c96a8f8 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -5,16 +5,27 @@ import json from pathlib import Path +import sys +from types import ModuleType import numpy as np import pytest import yaml from .. import native_plugin_builder +from . import official_reference from . import test_e2e as e2e from .benchmark import prepare_environment +def _write_stage1_text_encoder(model_dir: Path) -> Path: + stage1_text_encoder = model_dir / "stage1_text_encoder" + stage1_text_encoder.mkdir(parents=True) + for name in ("config.json", "tokenizer.json", "model.safetensors"): + (stage1_text_encoder / name).write_text("{}\n", encoding="utf-8") + return stage1_text_encoder + + def test_qualification_candidate_uses_the_prepared_family_model() -> None: profile = yaml.safe_load( (Path(__file__).parent / "benchmark/sana-wm-bidirectional.yaml").read_text(encoding="utf-8") @@ -121,6 +132,66 @@ def download(model_id, **kwargs): } +def test_official_reference_loads_stage1_encoder_from_explicit_local_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + stage1_text_encoder = tmp_path / "stage1-text-encoder" + stage1_text_encoder.mkdir() + for name in ("config.json", "tokenizer.json", "model.safetensors"): + (stage1_text_encoder / name).write_text("{}\n", encoding="utf-8") + calls: list[tuple[object, ...]] = [] + + class Decoder: + def to(self, device: str): + calls.append(("to", device)) + return "local-decoder" + + class Model: + def get_decoder(self): + calls.append(("get_decoder",)) + return Decoder() + + class AutoTokenizer: + padding_side = "left" + + @staticmethod + def from_pretrained(path: str, **kwargs): + calls.append(("tokenizer", path, kwargs)) + return AutoTokenizer() + + class AutoModelForCausalLM: + @staticmethod + def from_pretrained(path: str, **kwargs): + calls.append(("model", path, kwargs)) + return Model() + + transformers = ModuleType("transformers") + transformers.AutoModelForCausalLM = AutoModelForCausalLM + transformers.AutoTokenizer = AutoTokenizer + torch = ModuleType("torch") + torch.bfloat16 = "bf16" + monkeypatch.setitem(sys.modules, "transformers", transformers) + monkeypatch.setitem(sys.modules, "torch", torch) + + tokenizer, decoder = official_reference.load_local_stage1_text_encoder( + stage1_text_encoder, "cuda:0" + ) + assert isinstance(tokenizer, AutoTokenizer) + assert tokenizer.padding_side == "right" + assert decoder == "local-decoder" + assert calls == [ + ("tokenizer", str(stage1_text_encoder), {"local_files_only": True}), + ( + "model", + str(stage1_text_encoder), + {"local_files_only": True, "torch_dtype": "bf16"}, + ), + ("get_decoder",), + ("to", "cuda:0"), + ] + + def test_qualification_environment_replaces_gui_opencv(monkeypatch) -> None: calls = [] monkeypatch.setattr( @@ -196,6 +267,7 @@ def test_raw_snapshot_calls_declared_official_entrypoint(monkeypatch, tmp_path: (model_dir / "dit").mkdir(parents=True) (model_dir / "vae").mkdir() (model_dir / "refiner/text_encoder").mkdir(parents=True) + stage1_text_encoder = _write_stage1_text_encoder(model_dir) (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") assert not (model_dir / "model_index.json").exists() @@ -234,7 +306,9 @@ def decode(video_path: Path, frames_dir: Path): result = e2e._official_reference(model_dir, manifest, case, tmp_path) command = captured["command"] - assert command[1] == str(entrypoint) + assert command[1] == str(e2e.TEST_ROOT / "official_reference.py") + assert command[command.index("--reference-repo") + 1] == str(source) + assert command[command.index("--stage1-text-encoder") + 1] == str(stage1_text_encoder) assert command[command.index("--action") + 1] == case["action"] assert command[command.index("--intrinsics") + 1] == str( e2e._asset(case["camera_intrinsics_file"]) @@ -282,6 +356,7 @@ def test_official_reference_dependency_failure_is_not_hidden(monkeypatch, tmp_pa model_dir = tmp_path / "raw-checkpoint" (model_dir / "dit").mkdir(parents=True) (model_dir / "refiner/text_encoder").mkdir(parents=True) + _write_stage1_text_encoder(model_dir) (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") monkeypatch.setenv("TRTMC_REFERENCE_SOURCE_DIR", str(source)) @@ -338,6 +413,7 @@ def test_official_refiner_requires_every_output_frame( model_dir = tmp_path / "checkpoint" (model_dir / "dit").mkdir(parents=True) (model_dir / "refiner/text_encoder").mkdir(parents=True) + _write_stage1_text_encoder(model_dir) (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") monkeypatch.setenv("TRTMC_REFERENCE_SOURCE_DIR", str(source))