diff --git a/apps/benchmark/native/benchmark_worker.cpp b/apps/benchmark/native/benchmark_worker.cpp index 9af73aa010..cc26a83683 100644 --- a/apps/benchmark/native/benchmark_worker.cpp +++ b/apps/benchmark/native/benchmark_worker.cpp @@ -265,6 +265,7 @@ Json image_observation(const std::vector& results) { Json value = {{"generated_images", results.size()}, {"batch_size", results.size()}, {"generated_frames", frames}, + {"media_count", frames}, {"output_elements", pixels}}; if (!results.empty()) { value["height"] = results.front().height; @@ -276,39 +277,75 @@ Json image_observation(const std::vector& results) { return value; } +auto legacy_image_artifact_writer(std::string prefix) { + return [prefix = std::move(prefix), + iteration = std::uint64_t{0}](const std::vector& results) mutable { + ++iteration; + Json paths = Json::array(); + std::size_t artifact_index = 0; + for (const auto& result : results) { + if (result.height <= 0 || result.width <= 0 || result.channels <= 0 || + result.num_frames <= 0) + throw std::runtime_error("generated media has invalid dimensions"); + const auto frame_elements = + static_cast(result.height) * result.width * result.channels; + const auto expected = frame_elements * static_cast(result.num_frames); + if (expected != result.pixels.size()) + throw std::runtime_error("generated media has inconsistent pixel storage"); + for (int frame = 0; frame < result.num_frames; ++frame) { + const auto path = prefix + "." + std::to_string(iteration) + "." + + std::to_string(artifact_index++) + ".png"; + const auto offset = static_cast(frame_elements) * frame; + trtmc::cli::io::save_png( + path, {result.pixels.data() + offset, static_cast(frame_elements)}, + result.width, result.height, result.channels); + paths.push_back(std::filesystem::path(path).filename().string()); + } + } + return paths; + }; +} + Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& timing) { const auto config = image_config(request); - const std::string prompt = - request.at("prompt").is_array() ? "" : request.at("prompt").get(); + const bool batch_request = request.at("prompt").is_array(); + const std::string prompt = batch_request ? "" : request.at("prompt").get(); std::function()> invoke; std::optional cached; - if (auto* batch = dynamic_cast(&task)) { + if (batch_request) { + auto& batch = + require_interface(task, "IImageBatchGeneration"); const auto prompts = request.at("prompt").get>(); auto seeds = optional_value>(request, "seeds", {}); if (seeds.empty()) seeds.assign(prompts.size(), static_cast(std::max(config.seed, 0))); - invoke = [batch, prompts, seeds, config]() { - return batch->generate_image_batch(prompts, seeds, config); + invoke = [&batch, prompts, seeds, config]() { + return batch.generate_image_batch(prompts, seeds, config); }; - } else if (auto* edit = dynamic_cast(&task)) { + } else if (request.contains("image_path") && + dynamic_cast(&task) != nullptr) { + auto& edit = require_interface(task, "IImageEditing"); const std::string path = request.at("image_path").get(); if (!timing.asset_loading_included) cached = read_image(path); - invoke = [edit, prompt, path, config, &cached]() { + invoke = [&edit, prompt, path, config, &cached]() { if (cached) { - return std::vector{edit->generate_image( + return std::vector{edit.generate_image( prompt, cached->pixels.data(), cached->height, cached->width, config)}; } const Image image = read_image(path); - return std::vector{edit->generate_image( + return std::vector{edit.generate_image( prompt, image.pixels.data(), image.height, image.width, config)}; }; - } else if (auto* world = dynamic_cast(&task)) { + } else if (request.contains("image_path") && + dynamic_cast(&task) != nullptr) { + auto& world = + require_interface(task, "IWorldModelGeneration"); const std::string path = request.at("image_path").get(); if (!timing.asset_loading_included) cached = read_image(path); - invoke = [world, prompt, path, config, request, &cached]() { + invoke = [&world, prompt, path, config, request, &cached]() { std::optional loaded; if (!cached) loaded = read_image(path); @@ -323,7 +360,7 @@ Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& t optional_value>(request, "camera_intrinsics", {}); value.num_frames = optional_value(request, "num_frames", 0); value.generation = config; - return std::vector{world->generate_world(value)}; + return std::vector{world.generate_world(value)}; }; } else { auto& image = require_interface(task, "IImageGeneration"); @@ -331,7 +368,15 @@ Json run_generate_image(trtmc::ITask& task, const Json& request, const Timing& t return std::vector{image.generate_image(prompt, config)}; }; } - return measure(timing, invoke, image_observation); + auto write_images = + legacy_image_artifact_writer(request.at("_artifact_prefix").get()); + return measure(timing, invoke, [&](const auto& results) { + auto output = image_observation(results); + output[output.value("media_type", "image") == "video" ? "frame_artifacts" + : "image_artifacts"] = + write_images(results); + return output; + }); } Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& timing) { @@ -342,9 +387,15 @@ Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& t optional_value(request, "talker_max_new_tokens", 0); config.seed = optional_value(request, "seed", -1); const std::string prompt = request.at("prompt").get(); + auto write_audio = [prefix = request.at("_artifact_prefix").get(), + iteration = std::uint64_t{0}](const trtmc::AudioResult& result) mutable { + const auto path = prefix + "." + std::to_string(++iteration) + ".wav"; + trtmc::cli::io::write_wav(result, path); + return std::filesystem::path(path).filename().string(); + }; return measure( timing, [&]() { return interface.generate_audio(prompt, config); }, - [](const trtmc::AudioResult& result) { + [&](const trtmc::AudioResult& result) { const double seconds = result.sample_rate > 0 ? static_cast(result.samples.size()) / result.sample_rate @@ -352,7 +403,8 @@ Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& t return Json{{"output_samples", result.samples.size()}, {"num_samples", result.samples.size()}, {"output_audio_seconds", seconds}, - {"sample_rate", result.sample_rate}}; + {"sample_rate", result.sample_rate}, + {"audio_artifact", write_audio(result)}}; }); } @@ -366,6 +418,12 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) { config.max_new_tokens = optional_value(request, "max_new_tokens", 50); config.seed = optional_value(request, "seed", -1); config.tail_frames = optional_value(request, "tail_frames", 0); + auto write_audio = [prefix = request.at("_artifact_prefix").get(), + iteration = std::uint64_t{0}](const trtmc::AudioResult& result) mutable { + const auto path = prefix + "." + std::to_string(++iteration) + ".wav"; + trtmc::cli::io::write_wav(result, path); + return std::filesystem::path(path).filename().string(); + }; return measure( timing, [&]() { @@ -379,7 +437,7 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) { audio.sample_rate), static_cast(audio.samples.size()) / audio.sample_rate}; }, - [](const auto& value) { + [&](const auto& value) { const auto& result = value.first; return Json{{"input_audio_seconds", value.second}, {"output_audio_seconds", @@ -388,7 +446,8 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) { : 0.0}, {"output_samples", result.samples.size()}, {"num_samples", result.samples.size()}, - {"sample_rate", result.sample_rate}}; + {"sample_rate", result.sample_rate}, + {"audio_artifact", write_audio(result)}}; }); } @@ -465,6 +524,75 @@ Json run_transcribe(trtmc::ITask& task, const Json& request, const Timing& timin }); } +template +void write_binary_artifact(const std::string& path, const T* values, std::uint64_t count) { + if (count > static_cast(std::numeric_limits::max()) / sizeof(T)) + throw std::runtime_error("binary artifact is too large"); + std::ofstream output(path, std::ios::binary); + output.exceptions(std::ios::badbit | std::ios::failbit); + output.write(reinterpret_cast(values), + static_cast(count * sizeof(T))); + output.close(); +} + +Json run_geometry(trtmc::ITask& task, const Json& request, const Timing& timing) { + auto& interface = require_interface(task, "IMonocularGeometry"); + const auto path = request.at("image_path").get(); + std::optional cached; + if (!timing.asset_loading_included) + cached = read_image(path); + auto invoke = [&]() { + std::optional loaded; + if (!cached) + loaded = read_image(path); + const auto& image = cached ? *cached : *loaded; + return interface.estimate_geometry(image.pixels.data(), image.height, image.width); + }; + std::optional last; + for (int index = 0; index < timing.warmup; ++index) + last = invoke(); + Json observations = Json::array(); + for (int index = 0; index < timing.iterations; ++index) { + last.reset(); + const auto started = Clock::now(); + auto result = invoke(); + const auto wall_ms = elapsed_ms(started); + last.emplace(std::move(result)); + observations.push_back({{"runtime_e2e_wall_ms", wall_ms}, + {"geometry_images", 1}, + {"geometry_pixels", last->depth.size()}}); + } + if (last->height <= 0 || last->width <= 0) + throw std::runtime_error("monocular geometry returned invalid dimensions"); + const auto pixels = static_cast(last->height) * last->width; + if (last->points.size() != pixels * 3 || last->depth.size() != pixels || + last->mask.size() != pixels) + throw std::runtime_error("monocular geometry returned inconsistent output shapes"); + const auto prefix = request.at("_artifact_prefix").get(); + write_binary_artifact(prefix + ".points.f32", last->points.data(), last->points.size()); + write_binary_artifact(prefix + ".depth.f32", last->depth.data(), last->depth.size()); + write_binary_artifact(prefix + ".mask.u8", last->mask.data(), last->mask.size()); + Json intrinsics = Json::array(); + for (int row = 0; row < 3; ++row) + intrinsics.push_back({last->intrinsics[row * 3], last->intrinsics[row * 3 + 1], + last->intrinsics[row * 3 + 2]}); + return {{"observations", std::move(observations)}, + {"output_summary", + {{"geometry_images", 1}, + {"geometry_pixels", pixels}, + {"height", last->height}, + {"width", last->width}, + {"point_shape", {last->height, last->width, 3}}, + {"valid_pixels", std::count(last->mask.begin(), last->mask.end(), std::uint8_t{1})}, + {"normalized_intrinsics", std::move(intrinsics)}, + {"units", "meters"}, + {"camera_axes", {"right", "down", "forward"}}, + {"intrinsics_coordinates", "normalized_uv"}, + {"points_artifact", prefix + ".points.f32"}, + {"depth_artifact", prefix + ".depth.f32"}, + {"valid_mask_artifact", prefix + ".mask.u8"}}}}; +} + Json run_segment(trtmc::ITask& task, const Json& request, const Timing& timing) { auto& interface = require_interface(task, "ISegmentation"); const Image image = read_image(request.at("image_path").get()); @@ -475,7 +603,8 @@ Json run_segment(trtmc::ITask& task, const Json& request, const Timing& timing) {"num_masks", 1}, {"height", result.height}, {"width", result.width}, - {"mask_pixels", result.mask.size()}}; + {"mask_pixels", result.mask.size()}, + {"mask", result.mask}}; }); } @@ -502,9 +631,19 @@ Json run_segment_prompted(trtmc::ITask& task, const Json& request, const Timing& }; } return measure(timing, invoke, [](const trtmc::PromptedSegmentationResult& result) { - return Json{{"segmented_images", 1}, {"generated_masks", result.num_masks}, - {"num_masks", result.num_masks}, {"height", result.height}, - {"width", result.width}, {"mask_pixels", result.masks.size()}}; + if (result.boxes.size() % 4U != 0U) + throw std::runtime_error("prompted segmentation returned incomplete boxes"); + Json boxes = Json::array(); + for (std::size_t offset = 0; offset < result.boxes.size(); offset += 4U) { + boxes.push_back({result.boxes[offset], result.boxes[offset + 1U], + result.boxes[offset + 2U], result.boxes[offset + 3U]}); + } + return Json{ + {"segmented_images", 1}, {"generated_masks", result.num_masks}, + {"num_masks", result.num_masks}, {"height", result.height}, + {"width", result.width}, {"mask_pixels", result.masks.size()}, + {"masks", result.masks}, {"iou_scores", result.iou_scores}, + {"boxes", std::move(boxes)}, {"box_coordinates", "original_image_pixels_xyxy"}}; }); } @@ -526,10 +665,23 @@ Json run_detect(trtmc::ITask& task, const Json& request, const Timing& timing) { return measure( timing, [&]() { return interface.detect(image.pixels.data(), image.height, image.width); }, [](const trtmc::ObjectDetectionResult& result) { + Json boxes = Json::array(), scores = Json::array(), classes = Json::array(); + for (const auto& detection : result.boxes) { + boxes.insert(boxes.end(), + {detection.x_min, detection.y_min, detection.x_max, detection.y_max}); + scores.push_back(detection.score); + classes.push_back(detection.class_id); + } return Json{{"detected_images", 1}, {"detections", result.boxes.size()}, {"image_height", result.image_height}, - {"image_width", result.image_width}}; + {"image_width", result.image_width}, + {"boxes", std::move(boxes)}, + {"scores", std::move(scores)}, + {"class_ids", std::move(classes)}, + {"shape", {result.boxes.size(), 4}}, + {"coordinates", "xyxy"}, + {"units", "pixels"}}; }); } @@ -545,6 +697,7 @@ Json run_extract_features(trtmc::ITask& task, const Json& request, const Timing& [](const trtmc::ImageFeaturesResult& result) { return Json{{"processed_images", 1}, {"last_hidden_state_shape", result.last_hidden_state_shape}, + {"pooler_output", result.pooler_output}, {"pooler_output_shape", result.pooler_output_shape}, {"feature_elements", result.last_hidden_state.size() + result.pooler_output.size()}}; @@ -613,10 +766,12 @@ Json run_embedding(trtmc::ITask& task, const Json& request, const Timing& timing auto& interface = require_interface(task, "IEncoding"); invoke = [&interface, prompt]() { return interface.encode(prompt); }; } - return measure(timing, invoke, [](const trtmc::EmbeddingResult& result) { + return measure(timing, invoke, [pooled](const trtmc::EmbeddingResult& result) { return Json{{"embedding_vectors", 1}, {"embedding_elements", result.data.size()}, - {"dim", result.dim}}; + {"dim", result.dim}, + {"values", result.data}, + {"feature_kind", pooled ? "pooled" : "token"}}; }); } @@ -683,6 +838,7 @@ Json run_control(trtmc::ITask& task, const Json& request, const Timing& timing) return Json{{"action_steps", result.num_actions}, {"action_dim", result.action_dim}, {"action_values", result.actions.size()}, + {"actions", result.actions}, {"within_training_bounds", result.within_training_bounds}, {"inference_ms", result.inference_ms}}; }); @@ -2867,17 +3023,6 @@ Json run_track_pose(const trtmc::Model& model, const Json& request, const Timing }); } -template -void write_binary_artifact(const std::string& path, const T* values, std::uint64_t count) { - if (count > static_cast(std::numeric_limits::max()) / sizeof(T)) - throw std::runtime_error("binary artifact is too large"); - std::ofstream output(path, std::ios::binary); - output.exceptions(std::ios::badbit | std::ios::failbit); - output.write(reinterpret_cast(values), - static_cast(count * sizeof(T))); - output.close(); -} - Json run_geometry(const trtmc::Model& model, const Json& request, const Timing& timing, const std::string& task_id) { const auto task = task_for_operation(model, task_id); @@ -3769,6 +3914,7 @@ Json execute(const Json& request, const std::string& output_path) { {"classify", run_classify}, {"detect", run_detect}, {"extract_features", run_extract_features}, + {"geometry", run_geometry}, {"disparity", run_disparity}, {"rerank", run_rerank}, {"encode", run_encode}, diff --git a/apps/benchmark/performance/README.md b/apps/benchmark/performance/README.md deleted file mode 100644 index 754f30c28d..0000000000 --- a/apps/benchmark/performance/README.md +++ /dev/null @@ -1,356 +0,0 @@ - - -# Performance matrix - -The performance matrix is an application above Model Connect: - -~~~text -performance matrix -> trtmc-bench -> public build and Task APIs -performance matrix -> reference runner -~~~ - -Neither the native core nor any family imports benchmark code. The candidate -worker selects the bundle's runtime mode before execution: migrated bundles use -the public header-only Task SDK and C ABI, while existing bundles use -load_task(bundle, runtime_root). Both call the family's implementation directly; -a failed SDK call is never retried through an old interface. - -## Task API benchmark - -Use the packaged CLI for a single model or the checked-in multi-model example: - -```bash -trtmc-bench list models -trtmc-bench run --model distilgpt2 --runtime-root /opt/trtmc/lib -o results/distilgpt2 -trtmc-bench run apps/benchmark/example.yaml -o results/example -``` - -Without another workload, `--model` uses that model's E2E testcase. To benchmark -user input, pass public Task request JSON or JSONL. Each JSONL row may be a raw -request or `{"name": "case-name", "request": {...}}`; relative `*_path` values -are resolved from the data file. - -```bash -trtmc-bench run --model gpt2-125m --data requests.jsonl \ - --runtime-root /opt/trtmc/lib -o results/gpt2-data -``` - -Missing bundles are built through the public build command and cached. Pass -`--no-build` when every selected bundle must already exist. - -Managed cache reuse requires a matching benchmark receipt for the resolved -immutable Hugging Face snapshot, manifest, build arguments, builder sources, -and build environment. Changed or missing identities rebuild the bundle; -`--no-build` reports an error instead. Arbitrary local checkpoint directories -are mutable and are rebuilt rather than assumed unchanged. An explicitly -supplied bundle outside the managed cache remains the caller's provenance -responsibility. `--rebuild` forces a fresh managed build. - -### Select another Task from the same bundle - -A family-owned testcase can select a secondary Task without changing the -manifest's primary `task` or rebuilding the bundle: - -```json -{ - "name": "token-features", - "selected_task": "text_to_token_features", - "inputs": {"token_ids": [7, 9]}, - "config": {} -} -``` - -The existing `--case token-features` option selects this workload. Alternatively, -`--task text_to_token_features` selects the interface for the chosen testcase; -its inputs must match that interface. The operation defaults from the selected -Task. An explicit incompatible operation is an error, not a request to try -another interface. `selected_task` is call selection, not family Config. - -The worker checks that the loaded model actually binds the requested Task. -Bundle identity and managed-cache checks still use the manifest's primary Task. -Results, reproduction records and history comparison retain the selected Task, -so token features and pooled features cannot become the same performance series. -Performance references receive the selection separately from the unchanged -manifest. Their input and output support must match the chosen contract; the -text-only reference loaders reject token IDs instead of replacing them with -empty text. This selection path requires a family migrated to the Task SDK. - -The `head_scores` operation uses `head-scores-shape` for performance comparison: -both outputs must contain every finite score and agree on tensor shape, score -kind, pooling and normalization. It does not compare score accuracy or turn -logits into embeddings. Numerical acceptance remains in the owning family's -correctness tests; a matching reference implementation is still required. - -The `geometry`, `predict_structure` and `refine_pose` operations similarly use -`metric-geometry-shape`, `molecular-structure-shape` and `pose-refinement-shape`. -These contracts check complete output fields and artifact lengths, semantic -layouts, confidence presence and pose callback metadata. Their receipts mark -`numerical_parity_checked: false`: predicted values and opaque serialization -lengths need not be identical across implementations. Existing family numerical -thresholds remain authoritative. - -`offline_speech_dialogue` uses `offline-speech-shape`: the complete candidate -event PCM and reference Float32 WAV must agree on input/output counts and audio -formats. Text is retained per epoch, with final text replacing partial updates -instead of duplicating them. The comparison records both texts but does not -judge text or audio accuracy; `text_parity_checked` and -`numerical_parity_checked` are false. It cannot qualify live or tool dialogue. - -### Family-owned performance declarations - -The canonical release suite also reads optional -`families//tests/performance.yaml` files using the same -`trtmc.perf-suite/v2` format. Each file may name only that family's manifests and -testcases. An owned entry with the same ID and family replaces a central entry -in place; other new IDs are appended. Replacement uses the complete owned -declaration and its own defaults, not a partial merge with the old entry. Retain -the original workload, measurement and comparison criteria explicitly when -migrating a reference. Already-expanded additional profiles require their own -exact-ID replacements. Cross-family ID conflicts, repeated IDs within a family -file and family exclusions remain errors. A standalone user suite does not -implicitly include other suites. Check, prepare, run and resume use the same -composition. -An accepted owned declaration activates only its explicitly named model from -central exclusions for validation; it does not establish qualification. - -Use an existing reference adapter when it implements the actual workload. -Otherwise declare a Python script relative to the owning family directory: - -```yaml -schema_version: trtmc.perf-suite/v2 -name: example-performance -defaults: - measurement: {warmup: 3, iterations: 10} -entries: - - id: example.head_scores - family: example - model: example-model - operation: head_scores - workload: {testcase: example-head-scores} - baseline: - runner: task-reference - script: tests/performance_reference.py - mode: pytorch-eager - timing_scope: task-model-call-wall - input_preparation_included: false - asset_loading_included: false -``` - -This is an illustrative declaration, not an existing qualified model. `script` -and `adapter` are mutually exclusive. The script must be an existing `.py` file -inside its family, without symlinks or parent-directory traversal. It runs through -the existing subprocess mechanism; the core does not import or execute it. - -The script receives the existing model/revision, family, operation, manifest, -request JSON, selected Task, adapter-options JSON, precision, mode, padding, -warmup, iterations, case name, output path and timing-contract JSON arguments. -When no mode is declared, a custom script uses the neutral label `reference`, -not an implied `torch.compile` claim. Supplied token IDs stay in request JSON; -the script must execute the requested operand and Task, not decode/re-tokenize or -substitute a different operation. Preparation needed by the candidate must happen -explicitly before the candidate runs; a reference is not a preparation hook. - -Return `trtmc.perf-baseline/v1` JSON with `status: completed`, matching model, -family, operation, case name, selected Task, precision and mode. Include exact -warmup/iteration counts in `measurement`; include the three declared timing -fields both at top level and in `measurement_policy`. Return one actual finite, -positive `samples_ms` value per measured invocation, its median in -`metrics.latency_ms.p50`, and the complete `output_summary` required by the output -contract. Loading happens once before warmup; input preparation and output -materialization follow the declared clock. Metadata validation does not prove -model accuracy or make an unexecuted reference qualified. - -## DataFrame forecast formatting - -Install `pandas` separately (`pip install pandas`) when using the optional table -helpers. They do not load a model or run Python inference: - -```python -import pandas as pd -from trtmc_benchmark.dataframe import prepare_forecast_frame, format_forecast_frame - -frame = pd.read_csv("history.csv", parse_dates=["ds"]) -prepared = prepare_forecast_frame(frame, freq="D") -# prepared.request is the existing native worker's batch solve payload. -# Pass it as request= to an existing resolved batch forecast case. -``` - -Input columns default to `unique_id`, `ds`, and `values`. Series keep first-seen -order; time is sorted within each series. Timestamps must match the explicit -calendar frequency: gaps/duplicates are errors, not silently resampled data. -Use `value_columns=("x", "y")` for one two-channel series, not two batch items. -Missing values retain their masks; no shared imputation or normalization occurs. -`config_by_series={"sensor-a": {"frequency": 0}}` passes family Config unchanged; -calendar `freq` never selects a model's frequency category. - -For an existing resolved `case` whose bundle declares one of -`batch_series_to_point_forecast`, `batch_series_to_quantile_forecast`, or -`batch_series_to_point_and_quantile_forecast`, reuse the normal worker: - -```python -from pathlib import Path -from trtmc_benchmark.types import MeasurementSpec -from trtmc_benchmark.worker import find_worker, run_worker - -case = case.with_values(request=prepared.request, - measurement=MeasurementSpec(warmup=0, iterations=1)) -output_dir = Path("forecast-results") -output_dir.mkdir(parents=True, exist_ok=True) -receipt = run_worker(case, output_dir, find_worker()) -forecast = format_forecast_frame(prepared, receipt["output_summary"]) -``` - -The case retains its explicit runtime root. This example executes one complete -native batch; normal benchmark warmup/iteration settings repeat that batch. -Scalar-only routes reject the batch payload rather than retrying each series. -Real family native-batch support must be qualified separately; this helper does -not imply that the existing TimesFM batch-one bundle supports native batching. - -The result keeps series/horizon/channel order, actual horizon offsets, output -channel labels/units, and all actual quantile levels. The `forecast` column is -the real point output, never a substituted median; quantile-only results have -no invented point column. Unknown output labels remain missing. Heterogeneous -quantile columns have missing cells where a series did not predict that level, -with actual per-series levels recorded in `forecast.attrs`. - -## Timing contract - -Candidate measurements use one scope: public_task_call_wall. - -The candidate runs each family's production Task implementation as loaded. The -benchmark does not override family runtime policy with tuning switches such as -CUDA Graph enablement. - -The new architecture transfers control directly from the public Task interface -to the family implementation, so the old pipeline-call and model-call scopes are -the same boundary. Bundle loading, worker startup, warmup, telemetry, report -generation, and bundle building are excluded. Asset loading is excluded unless a -case explicitly sets asset_loading_included to true. - -The release suite defaults to three warmups and ten measured iterations. A -reference within 5% of candidate p50 is equivalent. Candidate and reference -failures are operational failures, not red performance results. - -## Commands - -Point the environment at one installed runtime directory containing the runtime, -TensorRT backend, and selected family DSOs: - -~~~bash -export TRTMC_PERF_WORKER=/opt/trtmc/bin/trtmc_benchmark_worker -export TRTMC_PERF_RUNTIME_ROOT=/opt/trtmc/lib -export TRTMC_PERF_BUNDLE_CACHE=/data/trtmc-bundles -export TRTMC_PERF_BUNDLE_ROOTS=/data/prebuilt-bundles -export TRTMC_ELF_REFERENCE_REPO=/opt/references/ELF -export TRTMC_LANCE_REFERENCE_REPO=/opt/references/Lance -export TRTMC_SANA_WM_REFERENCE_REPO=/opt/references/Sana -export TRTMC_SANA_WM_MODEL_DIR=/models/sana-wm -export PERSONAPLEX_OFFICIAL_REPO=/opt/references/personaplex -export TRTMC_FAST_FOUNDATION_STEREO_MODEL_DIR=/models/fast-foundation-stereo - -python3 tools/perf_matrix.py check \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml - -python3 tools/perf_matrix.py run \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ - --entry gpt2.generate -~~~ - -Bundle preparation is a separate, untimed step: - -~~~bash -python3 tools/perf_matrix.py prepare \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ - --entry gpt2.generate \ - --output artifacts/perf/bundle-preparation.json -~~~ - -Continue an interrupted run and regenerate its report: - -~~~bash -python3 tools/perf_matrix.py resume artifacts/perf/ -python3 tools/perf_matrix.py report artifacts/perf/ -~~~ - -## Configuration - -The release suite and optional family-owned suites declare model, testcase, -operation, measurement, reference and comparison semantics. Machine paths live -in the existing environment YAML. - -A release entry has one explicit testcase: - -~~~yaml -- id: gpt2.generate - family: gpt2 - operation: generate - model: distilgpt2 - workload: - testcase: distilgpt2 - baseline: - runner: hf-transformers - mode: torch-compile -~~~ - -The catalog reads families/*/tests/manifests/*.json directly. It has no second -model registry; optional performance declarations reuse those exact manifests. -A task that the worker does not implement is reported as unsupported. - -The entry, model, and model-selection options select work. resume continues -incomplete rows, and report regenerates JSON and HTML from stored measurements. - -Reference runners are separate processes so their dependencies never enter the -candidate worker or core runtime. The benchmark-owned cache receipt protects -managed build reuse without changing the public bundle format. It does not -certify externally supplied bundle bytes. - -HTML reports include p50, p95, task rates, measurement boundaries, reproduction -commands, and recorded evidence. Historical v1 results can be rendered without -relabelling their original timing scope; they cannot be resumed as v2 runs. -Historical latency deltas appear only when the recorded comparison identities -match. A performance result does not establish model correctness. - -External reference checkouts must provide their official runtime dependencies. -In particular, PersonaPlex requires the real `sphn` package and Lance requires -the dependencies imported by its upstream `inference_lance.py`; the benchmark -does not install substitutes for either reference. Fast Foundation Stereo also -requires the image and OpenCV packages imported by its official source tree. - -Install benchmark-only dependencies without adding them to a model family: - -```bash -python -m pip install -r apps/benchmark/performance/requirements.txt -``` - -## Internal Accuracy and Performance qualification - -The installed `trtmc-bench` remains a user application. Repository CI and QA -use the separate, non-packaged `tools/model_benchmark.py` driver. A family opts -in by adding one YAML file under `families//tests/benchmark/`; the file -may contain multiple `accuracy` and `performance` cases. Families without that -file, including L0-only models, are not discovered. - -Run every discovered case or select an exact model: - -```bash -python3 tools/model_benchmark.py list -python3 tools/model_benchmark.py run --all --kind performance \ - --runtime-root /opt/trtmc/lib --worker /opt/trtmc/bin/trtmc_benchmark_worker -python3 tools/model_benchmark.py run --model gpt2-125m --kind accuracy \ - --dataset mmlu-five-shot=/data/mmlu_dataset.json \ - --runtime-root /opt/trtmc/lib --worker /opt/trtmc/bin/trtmc_benchmark_worker -``` - -Manual or restricted datasets are supplied as `--dataset ID=PATH`; public -download definitions may instead materialize into `--data-root`. Model files do -not select a GPU. Both Accuracy and Performance invoke the installed -`trtmc-bench` through its public bundle and Task path. Performance tries the -declared compiled reference first and uses eager only when that reference fails -to execute. A completed reference whose output disagrees with the candidate is -a failed contract and never triggers fallback. diff --git a/apps/benchmark/performance/baselines/elf_reference.py b/apps/benchmark/performance/baselines/elf_reference.py deleted file mode 100644 index 11790c49c1..0000000000 --- a/apps/benchmark/performance/baselines/elf_reference.py +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Run the official PyTorch ELF implementation with replayable sampling inputs.""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from pathlib import Path - -import numpy as np -import torch - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--reference-repo", required=True) - parser.add_argument("--config", required=True) - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--dataset", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--shared-inputs-dir", required=True) - parser.add_argument( - "--generation-mode", choices=("conditional", "unconditional"), required=True - ) - parser.add_argument("--sampling-method", choices=("ode", "sde"), required=True) - parser.add_argument("--num-steps", type=int, required=True) - parser.add_argument("--cfg-scale", type=float, required=True) - parser.add_argument("--self-cond-cfg-scale", type=float, required=True) - parser.add_argument("--sde-gamma", type=float, default=0.0) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--precision", choices=("fp16", "fp32", "bf16"), default="bf16") - parser.add_argument("--initial-latents", type=Path) - parser.add_argument("--sampling-steps", type=Path) - parser.add_argument("--condition-latents", type=Path) - parser.add_argument("--condition-mask", type=Path) - parser.add_argument("--sde-noises", type=Path) - parser.add_argument("--local-files-only", action="store_true") - return parser.parse_args() - - -def _load_rows(path: Path) -> list[dict]: - return [ - json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip() - ] - - -def _sampling_steps(torch_mod, *, count: int, mean: float, std: float, generator): - inner = torch_mod.randn((count - 1,), generator=generator, dtype=torch_mod.float32) - inner = torch_mod.sigmoid(inner * std + mean).sort().values - return torch_mod.cat((torch_mod.zeros(1), inner, torch_mod.ones(1))) - - -def _trim_terminal(token_ids: list[int], terminal_ids: set[int]) -> list[int]: - end = len(token_ids) - while end and token_ids[end - 1] in terminal_ids: - end -= 1 - return token_ids[:end] - - -def _read_float32(path: Path, *, expected: int | None = None) -> torch.Tensor: - values = np.fromfile(path, dtype=np.float32) - if expected is not None and values.size != expected: - raise ValueError(f"{path} contains {values.size} float32 values; expected {expected}") - return torch.from_numpy(values.copy()) - - -def _validate_replay_arguments(args: argparse.Namespace) -> bool: - sampling_paths = (args.initial_latents, args.sampling_steps) - if any(sampling_paths) and not all(sampling_paths): - raise ValueError("--initial-latents and --sampling-steps must be provided together") - condition_paths = (args.condition_latents, args.condition_mask) - if any(condition_paths) and not all(condition_paths): - raise ValueError("--condition-latents and --condition-mask must be provided together") - if args.generation_mode == "conditional" and args.initial_latents and not all(condition_paths): - raise ValueError("conditional replay requires condition latents and condition mask") - return bool(args.initial_latents) - - -class _InferenceOptimizer: - """Accept checkpoint optimizer state without installing training-only packages.""" - - def load_state_dict(self, _state) -> None: - return None - - -def main() -> int: - args = _parse_args() - replay_inputs = _validate_replay_arguments(args) - reference_src = Path(args.reference_repo).resolve() / "src" - sys.path.insert(0, str(reference_src)) - - from configs.config import load_config_from_yaml - from generation import _build_eval_model - from modules.model import ELF_models - from modules.t5_encoder import get_encoder - from utils.checkpoint_utils import load_checkpoint - from utils.data_utils import ( - get_dataloader, - get_pad_token_id, - load_jsonl_dataset, - ) - from utils.encoder_utils import encode_text - from utils.generation_utils import ( - _dlm_decode_batch, - mask_after_eos, - shift_left, - ) - from utils.sampling_utils import _ode_step, _forward_sample, restore_cond - from utils.train_utils import TrainState - - if args.local_files_only: - import os - - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ["TRANSFORMERS_OFFLINE"] = "1" - - config = load_config_from_yaml(args.config) - config.use_bf16 = args.precision == "bf16" - config.use_compile = False - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - if device.type == "cuda": - # cuDNN SDPA can request an impractically large workspace for ELF's - # 1,088-token XSum sequence on constrained-memory CUDA hosts. Prefer - # PyTorch's bounded-memory SDPA implementations for a reference run. - torch.backends.cuda.enable_cudnn_sdp(False) - torch.backends.cuda.enable_flash_sdp(False) - torch.backends.cuda.enable_mem_efficient_sdp(True) - torch.backends.cuda.enable_math_sdp(True) - encoder_model_id = config.encoder_model_name - tokenizer_model_id = config.tokenizer_name or config.encoder_model_name - tokenizer = __import__("transformers").AutoTokenizer.from_pretrained( - tokenizer_model_id, - local_files_only=args.local_files_only, - ) - pad_token_id = get_pad_token_id(tokenizer, config.pad_token) - terminal_ids = {int(pad_token_id), int(tokenizer.eos_token_id)} - - encoder = None - if replay_inputs: - replay_initial = _read_float32(args.initial_latents) - if replay_initial.numel() % int(config.max_length) != 0: - raise ValueError("initial replay latents do not divide evenly by config.max_length") - text_encoder_dim = replay_initial.numel() // int(config.max_length) - elif args.generation_mode == "conditional": - encoder_config, encoder = get_encoder(encoder_model_id, torch.float32) - encoder = encoder.to(device).eval() - for parameter in encoder.parameters(): - parameter.requires_grad_(False) - text_encoder_dim = encoder_config.d_model - else: - text_encoder_dim = 512 - - model = ELF_models[config.model]( - text_encoder_dim=text_encoder_dim, - max_length=config.max_length, - attn_drop=config.attn_dropout, - proj_drop=config.proj_dropout, - num_time_tokens=config.num_time_tokens, - num_self_cond_cfg_tokens=config.num_self_cond_cfg_tokens, - vocab_size=tokenizer.vocab_size, - num_model_mode_tokens=config.num_model_mode_tokens, - bottleneck_dim=config.bottleneck_dim, - ).to(device) - state = TrainState( - model=model, - optimizer=_InferenceOptimizer(), - lr_scheduler=None, - ema_params1=TrainState.init_ema(model), - step=0, - epoch=0, - dropout_generator=torch.Generator(device="cpu").manual_seed(args.seed), - ) - state, _ = load_checkpoint(args.checkpoint, state) - model = _build_eval_model(state, use_compile=False).to(device).eval() - dtype = next(model.parameters()).dtype - autocast_dtype = { - "fp16": torch.float16, - "bf16": torch.bfloat16, - "fp32": torch.float32, - }[args.precision] - - rows = _load_rows(Path(args.dataset)) - conditional_batches = None - if args.generation_mode == "conditional" and not replay_inputs: - dataset = load_jsonl_dataset( - args.dataset, tokenizer, input_key="input", output_key="output" - ) - conditional_batches = iter( - get_dataloader( - dataset, - batch_size=1, - shuffle=False, - num_workers=0, - drop_last=False, - max_seq_length=config.max_length, - pad_token_id=pad_token_id, - max_input_seq_length=config.max_input_length, - distributed=False, - ) - ) - - shared_root = Path(args.shared_inputs_dir) - shared_root.mkdir(parents=True, exist_ok=True) - responses: list[dict] = [] - for index, row in enumerate(rows): - sample_id = str(row.get("id", f"elf_{index:06d}")) - sample_dir = shared_root / sample_id - sample_dir.mkdir(parents=True, exist_ok=True) - if replay_inputs: - latent_count = int(config.max_length) * int(text_encoder_dim) - z = replay_initial.reshape(1, config.max_length, text_encoder_dim).clone() - t_steps = _read_float32(args.sampling_steps, expected=args.num_steps + 1) - sde_noises = [] - if args.sde_noises: - noise = _read_float32(args.sde_noises) - sde_noises = list( - noise.reshape(-1, config.max_length, text_encoder_dim).unbind(0) - ) - if args.generation_mode == "conditional": - cond_seq = _read_float32( - args.condition_latents, expected=latent_count - ).reshape(1, config.max_length, text_encoder_dim) - cond_mask = _read_float32( - args.condition_mask, expected=int(config.max_length) - ).reshape(1, config.max_length) - else: - cond_seq = torch.zeros_like(z) - cond_mask = torch.zeros((1, config.max_length), dtype=torch.float32) - else: - generator = torch.Generator(device="cpu").manual_seed(args.seed + index) - t_steps = _sampling_steps( - torch, - count=args.num_steps, - mean=float(config.denoiser_p_mean), - std=float(config.denoiser_p_std), - generator=generator, - ) - z = torch.randn( - (1, config.max_length, text_encoder_dim), - generator=generator, - dtype=torch.float32, - ) * float(config.denoiser_noise_scale) - sde_noises = [] - if args.sampling_method == "sde": - for _ in range(args.num_steps - 1): - sde_noises.append( - torch.randn(z.shape, generator=generator, dtype=torch.float32) - * float(config.denoiser_noise_scale) - ) - z.numpy().tofile(sample_dir / "initial_latents.f32") - t_steps.numpy().tofile(sample_dir / "sampling_steps.f32") - if sde_noises: - torch.stack(sde_noises).numpy().tofile(sample_dir / "sde_noises.f32") - - if args.generation_mode == "conditional" and not replay_inputs: - batch = next(conditional_batches) - input_ids = torch.from_numpy(np.asarray(batch["input_ids"])).to(device).long() - encoder_mask = ( - torch.from_numpy(np.asarray(batch["encoder_attention_mask"])).to(device).float() - ) - cond_mask = torch.from_numpy(np.asarray(batch["cond_seq_mask"])).to(device).float() - cond_seq = encode_text( - input_ids=input_ids, - attention_mask=encoder_mask, - encoder=encoder, - latent_mean=config.latent_mean, - latent_std=config.latent_std, - ).to(dtype) - elif not replay_inputs: - cond_seq = torch.zeros_like(z, dtype=dtype, device=device) - cond_mask = torch.zeros((1, config.max_length), dtype=dtype, device=device) - - z = z.to(device=device, dtype=dtype) - t_steps = t_steps.to(device=device, dtype=dtype) - cond_seq = cond_seq.to(device=device, dtype=dtype) - cond_mask = cond_mask.to(device=device, dtype=dtype) - z = restore_cond(z, cond_seq, cond_mask) - x_pred = restore_cond(torch.zeros_like(z), cond_seq, cond_mask) - step_kwargs = dict( - model=model, - config=config, - cfg_scale=args.cfg_scale, - self_cond_cfg_scale=args.self_cond_cfg_scale, - cond_seq=cond_seq, - cond_seq_mask=cond_mask, - ) - started = time.perf_counter() - with ( - torch.no_grad(), - torch.amp.autocast( - "cuda", - dtype=autocast_dtype, - enabled=device.type == "cuda" and args.precision != "fp32", - ), - ): - for step_index in range(len(t_steps) - 2): - t = float(t_steps[step_index].item()) - t_next = float(t_steps[step_index + 1].item()) - if args.sampling_method == "ode": - z, x_pred = _ode_step( - z=z, t=t, t_next=t_next, x_pred_prev=x_pred, **step_kwargs - ) - else: - h = t_next - t - alpha = max(0.0, min(1.0, 1.0 - args.sde_gamma * h)) - t_back = alpha * t - eps = sde_noises[step_index].to(device=device, dtype=dtype) - z_back = restore_cond(alpha * z + (1.0 - alpha) * eps, cond_seq, cond_mask) - t_batch = torch.full((1,), t_back, dtype=dtype, device=device) - v_pred, x_pred = _forward_sample( - z=z_back, t_batch=t_batch, x_pred_prev=x_pred, **step_kwargs - ) - z = z_back + (t_next - t_back) * v_pred - z, x_pred = _ode_step( - z=z, - t=float(t_steps[-2].item()), - t_next=float(t_steps[-1].item()), - x_pred_prev=x_pred, - **step_kwargs, - ) - if args.precision == "fp16" and device.type == "cuda": - batch_size = z.shape[0] - t_final = torch.full( - (batch_size,), float(t_steps[-1].item()), dtype=z.dtype, device=device - ) - scale = ( - torch.full( - (batch_size,), args.self_cond_cfg_scale, dtype=z.dtype, device=device - ) - if config.num_self_cond_cfg_tokens > 0 - else None - ) - z_input = torch.cat([z, torch.zeros_like(z)], dim=-1) - with torch.amp.autocast("cuda", dtype=torch.float16): - _, decoder_logits = model( - z_input, - t_final, - deterministic=True, - self_cond_cfg_scale=scale, - decoder_step_active=True, - ) - predicted = decoder_logits.argmax(dim=-1) - else: - predicted = _dlm_decode_batch( - z, model, float(t_steps[-1].item()), config, args.self_cond_cfg_scale - ) - if args.generation_mode == "conditional": - cond_len = cond_mask.to(torch.int32).sum(dim=1) - predicted = shift_left(predicted, cond_len, 0)[ - :, : config.max_length - config.max_input_length - ] - predicted = mask_after_eos( - predicted, eos_token_id=tokenizer.eos_token_id, pad_token_id=pad_token_id - ) - token_ids = _trim_terminal( - [int(token_id) for token_id in predicted[0].detach().cpu().tolist()], - terminal_ids, - ) - responses.append( - { - "sample_id": sample_id, - "output_text": tokenizer.decode(token_ids, skip_special_tokens=True).strip(), - "generated_token_ids": token_ids, - "wall_ms": (time.perf_counter() - started) * 1000.0, - "source": "hf_elf_torch", - "shared_inputs_dir": str(sample_dir), - "shared_sampling_inputs": { - "initial_latents": str( - args.initial_latents or sample_dir / "initial_latents.f32" - ), - "sampling_steps": str( - args.sampling_steps or sample_dir / "sampling_steps.f32" - ), - **({"sde_noises": str(sample_dir / "sde_noises.f32")} if sde_noises else {}), - }, - } - ) - - Path(args.output).write_text( - json.dumps({"responses": responses}, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/benchmark/performance/baselines/reference_support.py b/apps/benchmark/performance/baselines/reference_support.py deleted file mode 100644 index 3bebc0710a..0000000000 --- a/apps/benchmark/performance/baselines/reference_support.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Reference-only helpers kept outside Model Connect core and families.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import numpy as np - - -def preprocess_locateanything(image_path: str) -> dict[str, np.ndarray]: - from PIL import Image - - size = 448 - patch = 14 - image = Image.open(image_path).convert("RGB").resize((size, size), Image.Resampling.BICUBIC) - pixels = np.asarray(image, dtype=np.float32) / 255.0 - pixels = (pixels - np.asarray((0.5, 0.5, 0.5), dtype=np.float32)) / np.asarray( - (0.5, 0.5, 0.5), dtype=np.float32 - ) - chw = pixels.transpose(2, 0, 1) - grid = size // patch - patches = ( - chw.reshape(3, grid, patch, grid, patch) - .transpose(1, 3, 0, 2, 4) - .reshape(grid * grid, 3, patch, patch) - ) - return { - "pixel_values": patches.astype(np.float32), - "image_grid_hws": np.asarray([[grid, grid]], dtype=np.int32), - } - - -def configure_official_model_args( - model: Any, *, max_disparity: int, valid_iters: int -) -> None: - model.args.max_disp = max_disparity - model.args.valid_iters = valid_iters - model.args.normalize = True - - -def fast_foundation_stereo_model_dir(configured: str) -> Path: - root = Path(configured).expanduser().resolve() - required = ( - root / "core/foundation_stereo.py", - root / "core/submodule.py", - root / "weights/23-36-37/model_best_bp2_serialize.pth", - ) - if not all(path.is_file() for path in required): - raise RuntimeError("Fast Foundation Stereo reference model directory is incomplete") - return root diff --git a/apps/benchmark/performance/baselines/task_reference.py b/apps/benchmark/performance/baselines/task_reference.py deleted file mode 100644 index 849bd7045a..0000000000 --- a/apps/benchmark/performance/baselines/task_reference.py +++ /dev/null @@ -1,2955 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Benchmark task-aligned non-text model reference operations.""" - -from __future__ import annotations - -import argparse -import atexit -from dataclasses import dataclass -from datetime import datetime, timezone -import json -import importlib -import math -import os -from pathlib import Path -import platform -import shutil -import statistics -import subprocess -import sys -import tempfile -import time -from typing import Any, Callable, Mapping, Sequence - - -REPOSITORY = Path(__file__).resolve().parents[4] -for source_root in ( - REPOSITORY, - REPOSITORY / "core/builder", - REPOSITORY / "apps/benchmark", -): - if str(source_root) not in sys.path: - sys.path.insert(0, str(source_root)) - -from apps.benchmark.performance.baselines.timing_contracts import timing_contract # noqa: E402 -from apps.benchmark.performance.baselines.hf_transformers import _batch_prompt, flatten_config # noqa: E402 - -SYSTEM_PROMPT_QWEN3_OMNI = ( - "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " - "capable of perceiving auditory and visual inputs, as well as generating " - "text and speech." -) -QWEN3_OMNI_TEXT_CHAT_TEMPLATE = """{%- for message in messages %} -{{- '<|im_start|>' + message.role + '\n' }} -{%- if message.content is string %} -{{- message.content }} -{%- else %} -{%- for item in message.content %} -{%- if item.type == 'text' %}{{- item.text }}{%- endif %} -{%- endfor %} -{%- endif %} -{{- '<|im_end|>\n' }} -{%- endfor %} -{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}""" -MAGPIE_SPEAKER_ENCODER_REPO = "Edresson/Speaker_Encoder_H_ASP" -MAGPIE_SPEAKER_ENCODER_FILENAME = "pytorch_model.bin" -MAGPIE_SPEAKER_ENCODER_URL = ( - "https://huggingface.co/Edresson/Speaker_Encoder_H_ASP/resolve/main/pytorch_model.bin" -) -# The official offline example default, matching the current native profile. -VOICECHAT_SYSTEM_PROMPT = ( - "You are an AI voice assistant developed by NVIDIA. Your name is NVIDIA Voice Chat. " - "Answer in a spoken, conversational style rather than a written one. Do not repeat " - "the same sentence over and over again. Start the conversation by greeting the user." -) -ADAPTERS = ( - "hf-diffusers", - "hf-qwen3-omni", - "hf-transformers-asr", - "hf-transformers-embedding", - "hf-transformers-reranking", - "hf-transformers-tts", - "hf-transformers-vision", - "hf-transformers-vlm", - "nemo-asr", - "nemo-tts", - "nemo-voicechat", - "pytorch-lerobot-act", - "pytorch-personaplex", - "pytorch-timeseries", - "upstream-elf", - "upstream-fast-foundation-stereo", - "upstream-lance", - "upstream-moge", - "upstream-sana-wm", -) -PYTORCH_ADAPTERS = { - "nemo-asr", - "nemo-tts", - "nemo-voicechat", - "pytorch-lerobot-act", - "pytorch-personaplex", - "pytorch-timeseries", - "upstream-elf", - "upstream-fast-foundation-stereo", - "upstream-lance", - "upstream-moge", - "upstream-sana-wm", -} - - -@dataclass(frozen=True) -class Session: - """One loaded reference model and its repeatable timed operation.""" - - invoke: Callable[[], Mapping[str, Any]] - framework: str - timing_scope: str = "task-model-call-wall" - input_preparation_included: bool = False - asset_loading_included: bool = False - compile_evidence: dict[str, Any] | None = None - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--adapter", required=True, choices=ADAPTERS) - parser.add_argument("--family", required=True) - parser.add_argument("--operation", required=True) - parser.add_argument("--model", required=True) - parser.add_argument("--revision") - parser.add_argument("--manifest", required=True, type=Path) - parser.add_argument("--selected-task", help="Semantic Task selected independently of bundle identity") - parser.add_argument("--request-json", required=True) - parser.add_argument("--adapter-options-json", default="{}") - parser.add_argument("--timing-contract-json", default="{}") - parser.add_argument("--precision", required=True, choices=("fp16", "fp32", "bf16")) - parser.add_argument( - "--mode", required=True, choices=("hf-eager", "pytorch-eager", "torch-compile") - ) - parser.add_argument("--padding", default="longest") - parser.add_argument("--trust-remote-code", action="store_true") - parser.add_argument("--local-files-only", action="store_true") - parser.add_argument("--warmup", required=True, type=int) - parser.add_argument("--iterations", required=True, type=int) - parser.add_argument("--case-name", required=True) - parser.add_argument("--testcase-name", help="Selected manifest testcase, not the performance entry ID") - parser.add_argument("--output", required=True, type=Path) - return parser - - -def _json_object(raw: str, label: str) -> dict[str, Any]: - try: - value = json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError(f"{label} is not valid JSON: {exc}") from exc - if not isinstance(value, dict): - raise ValueError(f"{label} must contain an object") - return value - - -def _selected_task(arguments: argparse.Namespace) -> str: - selected = getattr(arguments, "selected_task", None) - if selected is None: - return json.loads(arguments.manifest.read_text(encoding="utf-8"))["task"] - if not isinstance(selected, str) or not selected or selected != selected.strip(): - raise ValueError("selected_task must be a nonempty Task ID without surrounding whitespace") - return selected - - -def _reference_checkout(checkout: str, *, repository: str) -> str: - path = Path(checkout).resolve() - if not path.is_dir(): - raise ValueError(f"{repository} checkout does not exist: {path}") - return str(path) - - -def _torch_dtype(torch_module: Any, precision: str) -> Any: - return { - "fp16": torch_module.float16, - "fp32": torch_module.float32, - "bf16": torch_module.bfloat16, - }[precision] - - -def _seed_all(torch_module: Any, seed: int) -> None: - import random - - import numpy as np - - random.seed(seed) - np.random.seed(seed) - torch_module.manual_seed(seed) - if torch_module.cuda.is_available(): - torch_module.cuda.manual_seed_all(seed) - - -def _request_seed(request: Mapping[str, Any], default: Any = 42) -> int: - value = request.get("seed", default) - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError("reference request seed must be an integer") - return value - - -def _bark_generation_options(request: Mapping[str, Any]) -> dict[str, int]: - max_new_tokens = request.get("max_new_tokens", 0) - if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): - raise ValueError("Bark request max_new_tokens must be an integer") - if max_new_tokens <= 0: - return {} - return {"semantic_max_new_tokens": max_new_tokens} - - -def _apply_magpie_generation_options(model: Any, request: Mapping[str, Any]) -> None: - max_new_tokens = request.get("max_new_tokens", 0) - if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): - raise ValueError("Magpie request max_new_tokens must be an integer") - if max_new_tokens > 0: - model.inference_parameters.max_decoder_steps = max_new_tokens - - -def _load_kwargs(arguments: argparse.Namespace, torch_module: Any) -> dict[str, Any]: - values: dict[str, Any] = { - "trust_remote_code": arguments.trust_remote_code, - "local_files_only": arguments.local_files_only, - "torch_dtype": _torch_dtype(torch_module, arguments.precision), - } - if arguments.revision: - values["revision"] = arguments.revision - return values - - -def _processor_kwargs(arguments: argparse.Namespace) -> dict[str, Any]: - values: dict[str, Any] = { - "trust_remote_code": arguments.trust_remote_code, - "local_files_only": arguments.local_files_only, - } - if arguments.revision: - values["revision"] = arguments.revision - return values - - -def _load_sam3_processor( - transformers: Any, - model: str, - processor_kwargs: Mapping[str, Any], -) -> Any: - return transformers.Sam3Processor.from_pretrained(model, **processor_kwargs) - - -def _cached_snapshot_path(repo_id: str, requested: str | None, marker_file: str) -> Path | None: - if Path(repo_id).exists(): - return Path(repo_id).resolve() - try: - from huggingface_hub import try_to_load_from_cache - - cached = try_to_load_from_cache( - repo_id=repo_id, - filename=marker_file, - revision=requested or "main", - ) - except (ImportError, OSError, ValueError): - return None - if isinstance(cached, str) and Path(cached).is_file(): - return Path(cached).parent.resolve() - return None - - -def _to_device(value: Any, device: Any, dtype: Any = None) -> Any: - if isinstance(value, Mapping): - return {name: _to_device(item, device, dtype) for name, item in value.items()} - if hasattr(value, "to"): - if dtype is not None and getattr(value, "is_floating_point", lambda: False)(): - return value.to(device=device, dtype=dtype) - return value.to(device) - return value - - -def _asset_path(arguments: argparse.Namespace, request: Mapping[str, Any], key: str) -> Path: - raw = str(request.get(key, "") or "") - if not raw: - raise ValueError(f"{arguments.family} reference requires request.{key}") - path = Path(raw) - if not path.is_absolute(): - path = arguments.manifest.resolve().parent.parent / path - if not path.is_file(): - raise FileNotFoundError(f"reference input does not exist: {path}") - return path - - -def _tensor_summary(value: Any) -> dict[str, Any]: - shape = [int(dim) for dim in value.shape] - return { - "shape": shape, - "element_count": int(value.numel()), - "finite": bool(value.isfinite().all().item()), - } - - -def _flatten_tensor_values(value: Any) -> list[float]: - flattened: list[float] = [] - - def visit(item: Any) -> None: - if isinstance(item, (list, tuple)): - for child in item: - visit(child) - else: - flattened.append(float(item)) - - visit(value.detach().float().cpu().tolist()) - return flattened - - -def _forecast_summary( - value: Any, task: str, quantile_levels: Sequence[float] = () -) -> dict[str, Any]: - summary = _tensor_summary(value) - summary["values"] = _flatten_tensor_values(value) - if task not in {"series_to_point_forecast", "series_to_quantile_forecast"}: - return summary - shape = summary["shape"] - if len(shape) not in {2, 3} or shape[0] != 1: - raise ValueError("forecast reference requires an explicit one-series batch axis") - if task == "series_to_point_forecast": - horizon, channels = shape[1], shape[2] if len(shape) == 3 else 1 - summary.update(shape=[horizon, channels], axes=["horizon", "channel"]) - else: - if len(shape) != 3 or len(quantile_levels) != shape[1]: - raise ValueError( - "quantile forecast reference requires exact checkpoint quantile levels" - ) - horizon = shape[2] - summary.update( - shape=[shape[1], horizon, 1], - axes=["quantile", "horizon", "channel"], - quantile_levels=list(quantile_levels), - ) - summary.update( - forecast_elements=summary["element_count"], horizon_steps=list(range(1, horizon + 1)) - ) - return summary - - -def _regression_values_summary(value: Any) -> dict[str, Any]: - summary = _tensor_summary(value) - shape = summary["shape"] - if len(shape) != 2 or shape[0] != 1 or shape[1] <= 0: - raise ValueError("deterministic regression requires one batch with a nonempty target axis") - if not summary["finite"]: - raise ValueError("deterministic regression target values must be finite") - return {"kind": "regression_values", "target_count": shape[1], - "regression_targets": shape[1], "parameter_elements": 0, - "values": value[0].detach().float().cpu().tolist(), "axes": ["target"], - "target_names": [], "target_units": []} - - -def _regression_summary( - output: Any, distribution: str, parameter_names: Sequence[str] -) -> dict[str, Any]: - if distribution not in {"normal", "student_t", "negative_binomial"}: - raise ValueError("regression reference requires a declared probability distribution") - if not isinstance(output, (tuple, list)) or len(output) != len(parameter_names) or not output: - raise ValueError("regression reference parameters do not match the declared distribution") - # Transformers names distribution parameters loc/df. The public Task - # contract names them location/degrees_of_freedom; never infer tuple order. - aliases = {"loc": "location", "df": "degrees_of_freedom"} - names = [aliases.get(name, name) for name in parameter_names] - required = { - "normal": {"location", "scale"}, - "student_t": {"degrees_of_freedom", "location", "scale"}, - "negative_binomial": {"total_count", "logits"}, - }[distribution] - if len(names) != len(required) or set(names) != required: - raise ValueError("regression reference parameter names do not match the declared distribution") - parameters = [] - targets = None - for name, value in zip(names, output): - shape = tuple(int(size) for size in value.shape) - if ( - len(shape) != 2 - or shape[0] != 1 - or shape[1] <= 0 - or (targets is not None and shape[1] != targets) - ): - raise ValueError( - "regression parameters must each have one batch and the same target axis" - ) - targets = shape[1] - if not _tensor_summary(value)["finite"]: - raise ValueError("regression reference returned nonfinite parameters") - parameters.append({"name": name, "values": value[0].detach().float().cpu().tolist()}) - return { - "distribution": distribution, - "target_count": targets, - "regression_targets": targets, - "parameter_elements": targets * len(parameters), - "axes": ["target"], - "parameters": parameters, - "target_names": [], - "target_units": [], - } - - -def _task_value( - arguments: argparse.Namespace, - request: Mapping[str, Any], - name: str, - default: Any = None, -) -> Any: - if name in request: - return request[name] - return default - - -def _load_tts( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") - import torch - - torch.use_deterministic_algorithms(True) - device = torch.device("cuda") - prompt = str(request.get("prompt", "")) - if arguments.family == "magpie_tts": - import fsspec - from huggingface_hub import hf_hub_download - from nemo.collections.tts.models import MagpieTTSModel - - speaker_revision = str(options.get("speaker_encoder_revision", "") or "") - if not speaker_revision: - raise ValueError("Magpie TTS requires adapter_options.speaker_encoder_revision") - speaker_checkpoint = hf_hub_download( - repo_id=MAGPIE_SPEAKER_ENCODER_REPO, - filename=MAGPIE_SPEAKER_ENCODER_FILENAME, - revision=speaker_revision, - local_files_only=arguments.local_files_only, - ) - original_fsspec_open = fsspec.open - - def offline_fsspec_open(path: Any, *args: Any, **kwargs: Any) -> Any: - normalized = str(path).split("?", 1)[0] - if normalized == MAGPIE_SPEAKER_ENCODER_URL: - path = speaker_checkpoint - return original_fsspec_open(path, *args, **kwargs) - - fsspec.open = offline_fsspec_open - model_archive = hf_hub_download( - repo_id=arguments.model, - filename="magpie_tts_multilingual_357m.nemo", - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - model = MagpieTTSModel.restore_from(restore_path=model_archive).eval().to(device) - _apply_magpie_generation_options(model, request) - seed = _request_seed( - request, - 42, - ) - - def invoke() -> Mapping[str, Any]: - _seed_all(torch, seed) - with torch.inference_mode(): - audio, length = model.do_tts(transcript=prompt, language="en", use_cfg=True) - sample_count = int(length.item()) if length.numel() else int(audio.numel()) - return {"audio_samples": sample_count, "sample_rate": 22_050} - - else: - from transformers import AutoProcessor, BarkModel - - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - BarkModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - inputs = _to_device(processor(prompt, return_tensors="pt"), device) - seed = _request_seed( - request, - 42, - ) - generation_options = _bark_generation_options(request) - - def invoke() -> Mapping[str, Any]: - _seed_all(torch, seed) - with torch.inference_mode(): - audio = model.generate(**inputs, **generation_options) - return { - "audio_samples": int(audio.numel()), - "sample_rate": int(model.generation_config.sample_rate), - } - - if arguments.family == "magpie_tts": - return Session( - invoke, - "nemo", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - return Session(invoke, "transformers") - - -def _load_nemo_asr_reference_model( - arguments: argparse.Namespace, - *, - device: Any, -) -> Any: - if ( - arguments.family == "nemotron_speech_streaming" - and "nemotron-3.5-asr-streaming" in arguments.model.lower() - ): - from apps.benchmark.performance.baselines.audio_reference import ( - load_nemotron35_asr_model, - ) - - return load_nemotron35_asr_model( - model=arguments.model, - revision=arguments.revision or "", - local_files_only=arguments.local_files_only, - device=str(device), - ) - - import nemo.collections.asr as nemo_asr - - return nemo_asr.models.ASRModel.from_pretrained( - arguments.model, - map_location="cpu", - ) - - -def _disable_nemo_asr_cuda_graphs(model: Any) -> bool: - """Disable NeMo's optional RNNT CUDA Graph decoder when available.""" - decoding = getattr(getattr(model, "decoding", None), "decoding", None) - disable_cuda_graphs = getattr(decoding, "disable_cuda_graphs", None) - if not callable(disable_cuda_graphs): - return False - return bool(disable_cuda_graphs()) - - -def _load_asr( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - import torch - from apps.benchmark.performance.baselines.audio_reference import ( - read_wav_float32, - resample_audio, - transcription_text, - write_wav_pcm16, - ) - - audio, sample_rate = read_wav_float32(str(_asset_path(arguments, request, "audio_path"))) - target_rate = 16_000 - audio = resample_audio(audio, sample_rate, target_rate) - max_new_tokens = int(request.get("max_new_tokens", 100)) - device = torch.device("cuda") - - if arguments.family in {"canary", "nemotron_speech_streaming"}: - model = _load_nemo_asr_reference_model(arguments, device=device).eval().to(device) - _disable_nemo_asr_cuda_graphs(model) - reference_dtype = _torch_dtype(torch, arguments.precision) - autocast_dtype = reference_dtype if arguments.precision != "fp32" else torch.float16 - temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) - temporary.close() - atexit.register(Path(temporary.name).unlink, missing_ok=True) - write_wav_pcm16(Path(temporary.name), audio, target_rate) - - transcription_input: str | list[str] - if arguments.family == "nemotron_speech_streaming": - manifest = tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) - manifest.close() - atexit.register(Path(manifest.name).unlink, missing_ok=True) - record: dict[str, Any] = { - "audio_filepath": temporary.name, - "duration": float(len(audio)) / target_rate, - "text": "", - } - language = str(request.get("language", "") or "") - if language and language != "auto": - record["lang"] = language - Path(manifest.name).write_text(json.dumps(record) + "\n", encoding="utf-8") - transcription_input = manifest.name - else: - transcription_input = [temporary.name] - - def invoke() -> Mapping[str, Any]: - with torch.autocast( - device_type="cuda", - dtype=autocast_dtype, - enabled=arguments.precision != "fp32", - ): - result = model.transcribe(transcription_input, batch_size=1) - return {"text": transcription_text(result), "output_tokens": None} - - else: - from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor - - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - AutoModelForSpeechSeq2Seq.from_pretrained( - arguments.model, **_load_kwargs(arguments, torch) - ) - .eval() - .to(device) - ) - processor_options: dict[str, Any] = { - "sampling_rate": target_rate, - "return_tensors": "pt", - } - language = str(request.get("language", "") or "") - if language: - processor_options["language"] = language - inputs = _to_device( - processor(audio, **processor_options), device, next(model.parameters()).dtype - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - generated = model.generate(**inputs, max_new_tokens=max_new_tokens) - sequences = generated.sequences if hasattr(generated, "sequences") else generated - token_ids = [int(token) for token in sequences[0].detach().cpu().tolist()] - return { - "text": processor.batch_decode(sequences, skip_special_tokens=True)[0], - "token_ids": token_ids, - "output_tokens": len(token_ids), - } - - framework = ( - "nemo" if arguments.family in {"canary", "nemotron_speech_streaming"} else "transformers" - ) - if arguments.family in {"canary", "nemotron_speech_streaming"}: - return Session( - invoke, - framework, - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=True, - ) - return Session(invoke, framework) - - -def _load_vlm_model( - transformers_module: Any, family: str, model_id: str, kwargs: dict[str, Any] -) -> Any: - class_name = { - "internvl": "AutoModel", - "phi4_multimodal": "AutoModelForCausalLM", - "qwen_vl": "AutoModelForImageTextToText", - }[family] - model_class = getattr(transformers_module, class_name) - return model_class.from_pretrained(model_id, **kwargs) - - -def _load_deepseek_ocr( - arguments: argparse.Namespace, - request: Mapping[str, Any], -) -> Session: - import torch - from transformers import AutoModel, AutoTokenizer - - if arguments.precision != "bf16": - raise ValueError("DeepSeek-OCR-2 official reference requires bf16 precision") - tokenizer = AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - load_options = _load_kwargs(arguments, torch) - load_options.update( - { - "attn_implementation": "eager", - "use_safetensors": True, - } - ) - model = AutoModel.from_pretrained(arguments.model, **load_options).eval().cuda() - image_path = str(_asset_path(arguments, request, "image_path")) - prompt = str(request.get("prompt", "")) - if "" not in prompt: - prompt = f"\n{prompt}" - output_directory = Path(tempfile.mkdtemp(prefix="trtmc-perf-deepseek-ocr-")) - atexit.register(shutil.rmtree, output_directory, ignore_errors=True) - - def invoke() -> Mapping[str, Any]: - text = model.infer( - tokenizer, - prompt=prompt, - image_file=image_path, - output_path=str(output_directory), - base_size=1024, - image_size=768, - crop_mode=True, - save_results=False, - eval_mode=True, - ) - decoded = str(text or "") - token_ids = tokenizer(decoded, add_special_tokens=False).input_ids - return {"text": decoded, "token_ids": token_ids, "output_tokens": len(token_ids)} - - return Session( - invoke, - "transformers", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=True, - ) - - -def _locateanything_config(arguments: argparse.Namespace, transformers_module: Any) -> Any: - options = _processor_kwargs(arguments) - return transformers_module.AutoConfig.from_pretrained(arguments.model, **options) - - -def _locateanything_tokenizer(arguments: argparse.Namespace, _torch_module: Any) -> Any: - from transformers import AutoTokenizer - - return AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - - -def _load_locateanything( - arguments: argparse.Namespace, - request: Mapping[str, Any], -) -> Session: - import torch - import transformers - from apps.benchmark.performance.baselines.reference_support import ( - preprocess_locateanything, - ) - - if torch.cuda.is_available(): - torch.backends.cudnn.enabled = False - device = torch.device("cuda") - config = _locateanything_config(arguments, transformers) - tokenizer = _locateanything_tokenizer(arguments, torch) - load_options = _load_kwargs(arguments, torch) - load_options["config"] = config - model = ( - transformers.AutoModel.from_pretrained(arguments.model, **load_options).eval().to(device) - ) - image_inputs = preprocess_locateanything(str(_asset_path(arguments, request, "image_path"))) - pixel_values = torch.from_numpy(image_inputs["pixel_values"]).to(device) - image_grid_hws = torch.from_numpy(image_inputs["image_grid_hws"]).to( - device=device, dtype=torch.int32 - ) - prompt = str(request.get("prompt", "")) - prompt_text = ( - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - "<|im_start|>user\n" - + "" * 256 - + f"{prompt}<|im_end|>\n<|im_start|>assistant\n" - ) - inputs = tokenizer(prompt_text, return_tensors="pt") - input_ids = inputs["input_ids"].to(device) - generate_options: dict[str, Any] = { - "pixel_values": pixel_values, - "image_grid_hws": image_grid_hws, - "input_ids": input_ids, - "tokenizer": tokenizer, - "max_new_tokens": int(request.get("max_new_tokens", 32)), - "use_cache": True, - "generation_mode": "slow", - "do_sample": False, - } - if inputs.get("attention_mask") is not None: - generate_options["attention_mask"] = inputs["attention_mask"].to(device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - output = model.generate(**generate_options) - if isinstance(output, str): - text = output - token_ids = tokenizer.encode(text, add_special_tokens=False) - elif isinstance(output, (list, tuple)) and output and isinstance(output[0], str): - text = output[0] - token_ids = tokenizer.encode(text, add_special_tokens=False) - else: - sequence = output[0] if output.ndim > 1 else output - generated = sequence[input_ids.shape[-1] :] - token_ids = [int(token) for token in generated.detach().cpu().tolist()] - text = tokenizer.decode(token_ids, skip_special_tokens=True) - if not text.strip(): - raise RuntimeError("LocateAnything reference produced empty text") - return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} - - return Session(invoke, "transformers") - - -def _vl_prompt_has_image_placeholder(text: str) -> bool: - return any( - marker in text - for marker in ( - "<|image_pad|>", - "<|vision_start|>", - "<|image_1|>", - "", - "", - ) - ) - - -def _load_vlm( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - if arguments.family == "deepseek_ocr": - return _load_deepseek_ocr(arguments, request) - if arguments.family == "locateanything": - return _load_locateanything(arguments, request) - - import torch - import transformers - from PIL import Image - from transformers import AutoProcessor - - device = torch.device("cuda") - processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - load_options = _load_kwargs(arguments, torch) - if arguments.family == "phi4_multimodal": - config = transformers.AutoConfig.from_pretrained( - arguments.model, **_processor_kwargs(arguments) - ) - config._attn_implementation = "eager" - config._attn_implementation_internal = "eager" - load_options.update({"config": config, "attn_implementation": "eager"}) - model = ( - _load_vlm_model(transformers, arguments.family, arguments.model, load_options) - .eval() - .to(device) - ) - image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") - prompt = str(request.get("prompt", "")) - max_new_tokens = int(request.get("max_new_tokens", 16)) - - if arguments.family == "phi4_multimodal": - messages = [{"role": "user", "content": f"<|image_1|>{prompt}"}] - template_owner = getattr(processor, "tokenizer", processor) - else: - messages = [ - { - "role": "user", - "content": [ - {"type": "image", "image": image}, - {"type": "text", "text": prompt}, - ], - } - ] - template_owner = processor - rendered = template_owner.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True - ) - if not isinstance(rendered, str) or not _vl_prompt_has_image_placeholder(rendered): - raise ValueError("vision-language chat template lost the image placeholder") - inputs = processor(text=rendered, images=image, padding=True, return_tensors="pt") - inputs = _to_device(inputs, device, next(model.parameters()).dtype) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - output_ids = model.generate( - **inputs, - max_new_tokens=max_new_tokens, - do_sample=False, - num_beams=1, - ) - input_length = int(inputs["input_ids"].shape[-1]) - sequence = output_ids[0] - generated = sequence[input_length:] if sequence.shape[0] > input_length else sequence - token_ids = [int(token) for token in generated.detach().cpu().tolist()] - text = processor.decode(generated, skip_special_tokens=True) - return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} - - return Session(invoke, "transformers") - - -def _load_embedding( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - configured = _json_object(getattr(arguments, "timing_contract_json", "{}"), "--timing-contract-json") - if not configured: - raise ValueError("embedding reference requires explicit --timing-contract-json") - declared_timing = timing_contract(runner="task-reference", declared=configured) - if declared_timing["asset_loading_included"]: - raise ValueError("embedding reference preloads its assets; asset_loading_included must be false") - prompts = _batch_prompt(request) - if len(prompts) != 1: - raise ValueError("embedding reference requires exactly one text input") - import torch - from transformers import AutoModel, AutoTokenizer - - device = torch.device("cuda") - tokenizer = AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) - model = ( - AutoModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - prompt = prompts[0] - - def prepare_inputs() -> Mapping[str, Any]: - return _to_device( - tokenizer(prompt, return_tensors="pt", truncation=True), - device, - ) - - prepared_inputs = None - if not declared_timing["input_preparation_included"]: - prepared_inputs = prepare_inputs() - - def invoke() -> Mapping[str, Any]: - inputs = prepare_inputs() if prepared_inputs is None else prepared_inputs - with torch.inference_mode(): - outputs = model(**inputs, output_hidden_states=True) - hidden = getattr(outputs, "last_hidden_state", None) - if hidden is None: - hidden = outputs.hidden_states[-1] - mask = inputs.get("attention_mask", torch.ones(hidden.shape[:2], device=device)) - mask = mask.unsqueeze(-1).to(hidden.dtype) - vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) - vector = torch.nn.functional.normalize(vector, p=2, dim=-1) - summary = _tensor_summary(vector) - summary.update( - { - "embedding_vectors": 1, - "embedding_elements": int(vector.numel()), - "dim": int(vector.shape[-1]), - } - ) - return summary - - return Session( - invoke, - "transformers", - timing_scope=str(declared_timing["timing_scope"]), - input_preparation_included=bool(declared_timing["input_preparation_included"]), - asset_loading_included=False, - ) - - -def _load_reranking( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - import torch - from transformers import AutoModelForSequenceClassification, AutoProcessor - - device = torch.device("cuda") - documents = [str(document) for document in request.get("documents", [])] - query = str(request.get("query", "")) - model = ( - AutoModelForSequenceClassification.from_pretrained( - arguments.model, **_load_kwargs(arguments, torch) - ) - .eval() - .to(device) - ) - processor = AutoProcessor.from_pretrained( - arguments.model, - max_input_tiles=6, - use_thumbnail=True, - rerank_max_length=8192, - **_processor_kwargs(arguments), - ) - examples = [ - {"question": query, "doc_text": document, "doc_image": ""} for document in documents - ] - inputs = processor.process_queries_documents_crossencoder(examples) - inputs = _to_device(inputs, device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - logits = model(**inputs).logits.detach().float().cpu() - if logits.ndim == 2: - logits = logits[:, 0] if logits.shape[-1] == 1 else logits[:, -1] - scores = [float(score) for score in logits.reshape(-1).tolist()] - return {"scores": scores, "document_count": len(documents)} - - return Session(invoke, "transformers") - - -_PIXART_TRTMC_MIXED_PRECISION = "pixart_fp16_dit_fp32_t5" - - -def _diffusion_component_precision_contract( - arguments: argparse.Namespace, - options: Mapping[str, Any], -) -> str: - contract = str(options.get("component_precision_contract", "") or "") - if not contract: - return "" - if contract != _PIXART_TRTMC_MIXED_PRECISION: - raise ValueError(f"unsupported diffusion component precision contract: {contract}") - if arguments.family != "pixart" or arguments.precision != "fp16": - raise ValueError(f"{contract} requires family=pixart and precision=fp16") - return contract - - -def _cast_floating_tensors(value: Any, dtype: Any, torch_module: Any) -> Any: - if isinstance(value, torch_module.Tensor): - return value.to(dtype=dtype) if value.is_floating_point() else value - if isinstance(value, tuple): - return tuple(_cast_floating_tensors(item, dtype, torch_module) for item in value) - if isinstance(value, list): - return [_cast_floating_tensors(item, dtype, torch_module) for item in value] - if isinstance(value, dict): - return { - key: _cast_floating_tensors(item, dtype, torch_module) for key, item in value.items() - } - return value - - -def _configure_diffusion_component_precision( - pipeline: Any, - arguments: argparse.Namespace, - options: Mapping[str, Any], - torch_module: Any, -) -> None: - contract = _diffusion_component_precision_contract(arguments, options) - if not contract: - return - - def fp16_transformer_inputs(_module: Any, args: Any, kwargs: Any) -> Any: - return ( - _cast_floating_tensors(args, torch_module.float16, torch_module), - _cast_floating_tensors(kwargs, torch_module.float16, torch_module), - ) - - def fp32_transformer_output(_module: Any, _args: Any, output: Any) -> Any: - return _cast_floating_tensors(output, torch_module.float32, torch_module) - - pipeline.transformer.register_forward_pre_hook(fp16_transformer_inputs, with_kwargs=True) - pipeline.transformer.register_forward_hook(fp32_transformer_output) - - -def _diffusion_pipeline( - arguments: argparse.Namespace, - torch_module: Any, - options: Mapping[str, Any], -) -> Any: - import diffusers - - model_id = str(options.get("model_id", arguments.model)) - requested_revision = ( - str(options.get("model_revision", getattr(arguments, "revision", None) or "")) or None - ) - model_source: str | Path = model_id - if arguments.local_files_only: - cached = _cached_snapshot_path(model_id, requested_revision, "model_index.json") - if cached is None: - raise FileNotFoundError(f"cached Diffusers snapshot is missing: {model_id}") - model_source = cached - - load_options: dict[str, Any] = {} - component_contract = _diffusion_component_precision_contract(arguments, options) - if component_contract == _PIXART_TRTMC_MIXED_PRECISION: - from transformers import T5EncoderModel - - load_options["text_encoder"] = T5EncoderModel.from_pretrained( - model_source, - subfolder="text_encoder", - torch_dtype=torch_module.float32, - **( - {"revision": requested_revision} - if requested_revision and model_source == model_id - else {} - ), - local_files_only=arguments.local_files_only, - ) - if arguments.family == "wan2_2_ti2v": - load_options["vae"] = diffusers.AutoencoderKLWan.from_pretrained( - model_source, - subfolder="vae", - torch_dtype=torch_module.float32, - **( - {"revision": requested_revision} - if requested_revision and model_source == model_id - else {} - ), - local_files_only=arguments.local_files_only, - ) - return diffusers.DiffusionPipeline.from_pretrained( - model_source, - torch_dtype=_torch_dtype(torch_module, arguments.precision), - **load_options, - **( - {"revision": requested_revision} - if requested_revision and model_source == model_id - else {} - ), - trust_remote_code=bool(options.get("trust_remote_code", arguments.trust_remote_code)), - local_files_only=arguments.local_files_only, - ) - - -def _load_diffusers( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import inspect - import torch - from PIL import Image - - pipeline = _diffusion_pipeline(arguments, torch, options) - _configure_diffusion_component_precision(pipeline, arguments, options, torch) - if bool(options.get("cpu_offload", False)) and hasattr(pipeline, "enable_model_cpu_offload"): - pipeline.enable_model_cpu_offload() - else: - pipeline.to("cuda") - signature = inspect.signature(pipeline.__call__) - accepted = set(signature.parameters) - accepts_extra = any( - parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in signature.parameters.values() - ) - if "prompt" in request and "prompts" in request: - raise ValueError("reference request must not provide both prompt and prompts") - prompt = request.get("prompts", request.get("prompt", "")) - if "prompts" in request and not isinstance(prompt, list): - raise ValueError("prompts must contain one string per batch item") - if isinstance(prompt, list): - if not prompt or any(not isinstance(value, str) for value in prompt): - raise ValueError("prompt list must contain one string per batch item") - count = len(prompt) - else: - if not isinstance(prompt, str): - raise ValueError("prompt must be a string or a non-empty list of strings") - count = 1 - batch_size = request.get("batch_size", count) - if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - if isinstance(prompt, list): - if count != batch_size: - raise ValueError("prompt list must contain one string per batch item") - prompt_value: str | list[str] = list(prompt) - else: - prompt_value = [prompt] * batch_size if batch_size > 1 else prompt - values: dict[str, Any] = {"prompt": prompt_value} - negative_prompt = str(request.get("negative_prompt", "")) - if negative_prompt or "negative_prompt" in request or arguments.family == "qwen_image": - values["negative_prompt"] = negative_prompt - steps = int(request.get("num_steps", -1)) - if steps > 0: - values.update({"num_inference_steps": steps, "step": steps}) - manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) - height = int(request.get("height", request.get("video_height", manifest.get("image_height", 0)))) - width = int(request.get("width", request.get("video_width", manifest.get("image_width", 0)))) - if height > 0: - values["height"] = height - if width > 0: - values["width"] = width - num_frames = int( - request.get( - "video_num_frames", - _task_value(arguments, request, "num_frames", manifest.get("video_num_frames", 1)), - ) - ) - if num_frames > 0: - values["num_frames"] = num_frames - for name in ("action", "intrinsics"): - value = str(_task_value(arguments, request, name, "") or "") - if value: - values[name] = value - fps = int(_task_value(arguments, request, "fps", 0)) - if fps > 0: - values["fps"] = fps - for name in ("flow_shift", "translation_speed", "rotation_speed_deg"): - value = _task_value(arguments, request, name) - if value is not None: - values[name] = float(value) - cfg_scale = float(request.get("cfg_scale", -1.0)) - if cfg_scale >= 0 and arguments.family != "qwen_image": - values["cfg_scale"] = cfg_scale - if bool(request.get("no_refiner", False)): - values["no_refiner"] = True - guidance = float(request.get("guidance_scale", -1.0)) - if arguments.family == "qwen_image": - # Native Qwen guidance is true CFG; cfg_scale remains a legacy fallback. - true_cfg_scale = guidance if guidance >= 0 else cfg_scale - if true_cfg_scale >= 0: - values["true_cfg_scale"] = true_cfg_scale - elif guidance >= 0: - values["guidance_scale"] = guidance - values["output_type"] = "np" - if "image_path" in request and "image_paths" in request: - raise ValueError("reference request must not provide both image_path and image_paths") - image_path = request.get("image_path", "") - if "image_paths" in request: - image_paths = request["image_paths"] - if ( - not isinstance(image_paths, list) - or len(image_paths) != 1 - or not isinstance(image_paths[0], str) - or not image_paths[0] - ): - raise ValueError("this reference supports exactly one conditioning image") - image_path = image_paths[0] - task = getattr(arguments, "selected_task", None) or manifest["task"] - if task in {"images_text_to_image_edit", "image_edit"} and not image_path: - raise ValueError("image edit reference requires one conditioning image") - if image_path: - if "image" not in accepted and not accepts_extra: - raise ValueError("selected reference pipeline does not accept conditioning image") - values["image"] = Image.open( - _asset_path(arguments, {"image_path": image_path}, "image_path") - ).convert("RGB") - call_values = { - name: value for name, value in values.items() if name in accepted or accepts_extra - } - required = [str(name) for name in options.get("required_call_arguments", [])] - missing = [name for name in required if name not in call_values] - if missing: - raise ValueError( - f"{arguments.family} reference is missing required call arguments: " - + ", ".join(missing) - ) - seed = int(request.get("seed", 42)) - raw_seeds = request.get("seeds") - if raw_seeds is not None: - if not isinstance(raw_seeds, list) or len(raw_seeds) != batch_size: - raise ValueError("seeds must contain one integer per batch item") - seeds: int | list[int] = [int(value) for value in raw_seeds] - elif batch_size > 1: - seeds = [seed] * batch_size - else: - seeds = seed - if "generator" in accepted or accepts_extra: - if isinstance(seeds, list): - call_values["generator"] = [ - torch.Generator("cuda").manual_seed(value) for value in seeds - ] - else: - call_values["generator"] = torch.Generator("cuda").manual_seed(seeds) - - def invoke() -> Mapping[str, Any]: - if "generator" in call_values: - generators = call_values["generator"] - if isinstance(generators, list): - for generator, value in zip(generators, seeds, strict=True): - generator.manual_seed(value) - else: - generators.manual_seed(seeds) - result = pipeline(**call_values) - media = getattr(result, "images", None) - if media is None: - media = getattr(result, "frames", None) - media_type = str(request.get("media_type", "image")) - return _media_summary(media, media_type) - - return Session( - invoke, - "diffusers", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - - -def _media_count(media: Any, media_type: str) -> int: - """Count image batches or video frames without coercing arrays to bool.""" - if media is None: - return 0 - try: - outer_count = len(media) - except TypeError: - return 1 - if outer_count == 0 or media_type != "video": - return outer_count - try: - return len(media[0]) - except TypeError: - return outer_count - - -def _media_summary(media: Any, media_type: str) -> dict[str, Any]: - """Describe materialized image/video geometry without copying its pixels.""" - count = _media_count(media, media_type) - item = media - try: - item = item[0] - if media_type == "video": - item = item[0] - except (IndexError, KeyError, TypeError): - pass - width = height = channels = None - size = getattr(item, "size", None) - bands = getattr(item, "getbands", None) - if isinstance(size, tuple) and len(size) == 2 and callable(bands): - width, height = (int(value) for value in size) - channels = len(bands()) - else: - shape = tuple(int(value) for value in getattr(item, "shape", ())) - if len(shape) >= 3: - if shape[-1] in {1, 3, 4}: - height, width, channels = shape[-3:] - elif shape[-3] in {1, 3, 4}: - channels, height, width = shape[-3:] - summary = { - "media_type": media_type, - "media_count": count, - "height": height, - "width": width, - "channels": channels, - } - try: - import numpy as np - - array = np.asarray(media) - if np.issubdtype(array.dtype, np.number): - finite = bool(np.isfinite(array).all()) - if not finite: - raise RuntimeError("reference returned non-finite media values") - summary["finite"] = True - except (TypeError, ValueError): - pass - return summary - - -def _numeric_values(request: Mapping[str, Any], key: str) -> list[float]: - raw = request.get(key) - if not isinstance(raw, list) or not raw: - raise ValueError(f"time-series reference requires non-empty request.{key}") - return [float(value) for value in raw] - - -def _observed_values(request: Mapping[str, Any], count: int) -> list[float]: - raw = request.get("observed_mask") - if raw is None or raw == []: - return [1.0] * count - values = _numeric_values(request, "observed_mask") - if len(values) != count: - raise ValueError("time-series observed_mask must match past_values") - return values - - -def _align(values: Sequence[float], length: int, fill: float) -> list[float]: - result = [fill] * length - count = min(len(values), length) - result[-count:] = values[-count:] - return result - - -def _patchtst_task(config: Any) -> str: - for attribute in ("patchtst_task", "task_type", "problem_type"): - value = str(getattr(config, attribute, "") or "").lower() - if "class" in value: - return "classification" - if "regress" in value: - return "regression" - if "forecast" in value or "predict" in value: - return "forecast" - - architectures = getattr(config, "architectures", []) or [] - if isinstance(architectures, str): - architectures = [architectures] - for architecture in architectures: - value = str(architecture).lower() - if "class" in value: - return "classification" - if "regress" in value: - return "regression" - if "forecast" in value or "predict" in value: - return "forecast" - return "regression" - - -def _load_timeseries( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - import torch - import transformers - - device = torch.device("cuda") - dtype = _torch_dtype(torch, arguments.precision) - task_id = _selected_task(arguments) - if arguments.family == "chronos_bolt": - from chronos import ChronosBoltPipeline - - chronos_options = _processor_kwargs(arguments) - chronos_options.update({"device_map": str(device), "dtype": dtype}) - model = ChronosBoltPipeline.from_pretrained(arguments.model, **chronos_options) - compile_evidence = None - if arguments.mode == "torch-compile": - compile_evidence = _compile_forward(model.model) - raw = _numeric_values(request, "past_values") - observed = _observed_values(request, len(raw)) - context = torch.tensor([value if mask > 0 else float("nan") - for value, mask in zip(raw, observed)], dtype=dtype, device=device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - value = model.predict( - context, - prediction_length=model.model_prediction_length, - limit_prediction_length=True, - ) - quantiles = model.model.config.chronos_config["quantiles"] - return _forecast_summary(value, task_id, quantiles) - - return Session(invoke, "chronos", compile_evidence=compile_evidence) - - config = transformers.AutoConfig.from_pretrained( - arguments.model, **_processor_kwargs(arguments) - ) - if arguments.family == "timesfm": - model = ( - transformers.TimesFmModelForPrediction.from_pretrained( - arguments.model, **_load_kwargs(arguments, torch) - ) - .eval() - .to(device) - ) - length = int(model.config.context_length) - raw = _numeric_values(request, "past_values") - series = torch.tensor(_align(raw, length, 0.0), dtype=dtype, device=device).reshape( - 1, length - ) - observed = _align(_observed_values(request, len(raw)), length, 0.0) - padding = [0 if mask > 0 else 1 for mask in observed] - padding_tensor = torch.tensor(padding, dtype=torch.int32, device=device).reshape(1, length) - frequency = int(request.get("frequency", 0)) - frequency_tensor = torch.tensor([[frequency]], dtype=torch.long, device=device) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - decoder = model.decoder( - past_values=series, - past_values_padding=padding_tensor, - freq=frequency_tensor, - output_attentions=False, - output_hidden_states=False, - ) - output = model._postprocess_output( - decoder.last_hidden_state, (decoder.loc, decoder.scale) - )[:, -1, : model.config.horizon_length, 0] - return _forecast_summary(output, task_id) - - else: - is_mixer = arguments.family == "patchtsmixer" - if is_mixer: - model_class = transformers.PatchTSMixerForPrediction - output_name = "prediction_outputs" - else: - task = _patchtst_task(config) - class_name, output_name = { - "classification": ( - "PatchTSTForClassification", - "prediction_logits", - ), - "forecast": ("PatchTSTForPrediction", "prediction_outputs"), - "regression": ("PatchTSTForRegression", "regression_outputs"), - }[task] - model_class = getattr(transformers, class_name) - if not hasattr(model_class, "all_tied_weights_keys"): - model_class.all_tied_weights_keys = {} - model = ( - model_class.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) - .eval() - .to(device) - ) - length = int(config.context_length) - channels = int(config.num_input_channels) - raw = _numeric_values(request, "past_values") - values = torch.tensor( - _align(raw, length * channels, 0.0), dtype=dtype, device=device - ).reshape(1, length, channels) - aligned_mask = _align(_observed_values(request, len(raw)), length * channels, 0.0) - observed = torch.tensor( - aligned_mask, - dtype=dtype, - device=device, - ).reshape(1, length, channels) - - def invoke() -> Mapping[str, Any]: - with ( - torch.inference_mode(), - torch.autocast( - device_type="cuda", - dtype=dtype, - enabled=dtype != torch.float32, - ), - ): - if is_mixer: - outputs = model( - past_values=values * observed, - observed_mask=observed, - return_loss=False, - return_dict=True, - ) - output = outputs.prediction_outputs - else: - outputs = model( - past_values=values, - past_observed_mask=observed.gt(0.5), - return_dict=True, - ) - output = getattr(outputs, output_name) - if task_id == "series_to_regression_distribution": - return _regression_summary(output, config.distribution_output, - tuple(model.distribution_output.args_dim)) - if task_id == "series_to_regression_values": - return _regression_values_summary(output) - if isinstance(output, (tuple, list)): - output = torch.stack(list(output), dim=-1) - return _forecast_summary(output, task_id) - - return Session(invoke, "transformers") - - -def _load_vision( - arguments: argparse.Namespace, - request: Mapping[str, Any], - _options: Mapping[str, Any], -) -> Session: - import torch - from PIL import Image - import transformers - - device = torch.device("cuda") - image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") - width, height = image.size - kwargs = _load_kwargs(arguments, torch) - processor_kwargs = _processor_kwargs(arguments) - - if arguments.family in { - "timm_densenet", - "timm_efficientnet", - "timm_inception", - "timm_mnasnet", - "timm_mobilenetv2", - "timm_mobilenetv3", - "timm_repvgg", - "timm_resnet", - "timm_vgg", - "timm_vit", - }: - import timm - from timm.data import create_transform, resolve_model_data_config - - model = timm.create_model(f"hf-hub:{arguments.model}", pretrained=True) - dtype = _torch_dtype(torch, arguments.precision) - model = model.eval().to(device=device, dtype=dtype) - transform = create_transform(**resolve_model_data_config(model), is_training=False) - inputs = transform(image).unsqueeze(0).to(device=device, dtype=dtype) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - logits = model(inputs) - return {"top_class": int(logits.argmax(dim=-1)[0]), **_tensor_summary(logits)} - - elif arguments.family == "detr": - processor = transformers.AutoImageProcessor.from_pretrained( - arguments.model, **processor_kwargs - ) - model = ( - transformers.AutoModelForObjectDetection.from_pretrained(arguments.model, **kwargs) - .eval() - .to(device) - ) - inputs = _to_device( - processor(images=image, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - target_sizes = torch.tensor([[height, width]], device=device) - results = processor.post_process_object_detection( - outputs, threshold=0.5, target_sizes=target_sizes - )[0] - return { - "detected_images": 1, - "detections": int(results["scores"].shape[0]), - "image_height": height, - "image_width": width, - } - - elif arguments.family == "dinov3": - processor = transformers.AutoImageProcessor.from_pretrained( - arguments.model, **processor_kwargs - ) - model = transformers.AutoModel.from_pretrained(arguments.model, **kwargs).eval().to(device) - inputs = _to_device( - processor(images=image, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - return { - "last_hidden_state_shape": _tensor_summary(outputs.last_hidden_state)["shape"], - "pooler_output_shape": _tensor_summary(outputs.pooler_output)["shape"], - } - - elif arguments.family == "segformer": - processor = transformers.AutoImageProcessor.from_pretrained( - arguments.model, **processor_kwargs - ) - model = ( - transformers.AutoModelForSemanticSegmentation.from_pretrained(arguments.model, **kwargs) - .eval() - .to(device) - ) - inputs = _to_device( - processor(images=image, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - logits = model(**inputs).logits - masks = logits.argmax(dim=1) - return {"mask_count": int(masks.shape[0]), **_tensor_summary(masks)} - - elif arguments.family == "sam3": - processor = _load_sam3_processor(transformers, arguments.model, processor_kwargs) - model = transformers.Sam3Model.from_pretrained(arguments.model, **kwargs).eval().to(device) - inputs = _to_device( - processor( - images=image, - text=str(request.get("prompt", "")), - return_tensors="pt", - ), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - original_sizes = inputs.get("original_sizes") - target_sizes = ( - original_sizes.cpu().tolist() if hasattr(original_sizes, "cpu") else original_sizes - ) - result = processor.post_process_instance_segmentation( - outputs, - threshold=0.5, - mask_threshold=0.5, - target_sizes=target_sizes, - )[0] - masks = result.get("masks") - num_masks = 0 if masks is None else int(masks.shape[0] if masks.ndim > 2 else 1) - return { - "num_masks": num_masks, - "height": height, - "width": width, - } - - else: - processor = transformers.SamProcessor.from_pretrained(arguments.model, **processor_kwargs) - model = transformers.SamModel.from_pretrained(arguments.model, **kwargs).eval().to(device) - points = [ - [ - [ - int(float(request.get("point_x", 0.5)) * width), - int(float(request.get("point_y", 0.5)) * height), - ] - ] - ] - inputs = _to_device( - processor(image, input_points=points, return_tensors="pt"), - device, - next(model.parameters()).dtype, - ) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - outputs = model(**inputs) - masks = processor.image_processor.post_process_masks( - outputs.pred_masks.cpu(), - inputs["original_sizes"].cpu(), - inputs["reshaped_input_sizes"].cpu(), - )[0] - height, width = (int(value) for value in masks.shape[-2:]) - num_masks = int(masks.numel() // (height * width)) - return { - "num_masks": num_masks, - "height": height, - "width": width, - } - - return Session(invoke, "transformers") - - -def _qwen3_omni_chat_inputs(processor: Any, conversation: Sequence[Mapping[str, Any]]) -> Any: - processor_template = getattr(processor, "chat_template", None) - tokenizer_template = getattr(getattr(processor, "tokenizer", None), "chat_template", None) - template = ( - processor_template - if isinstance(processor_template, str) and processor_template.strip() - else tokenizer_template - if isinstance(tokenizer_template, str) and tokenizer_template.strip() - else QWEN3_OMNI_TEXT_CHAT_TEMPLATE - ) - return processor.apply_chat_template( - conversation, - chat_template=template, - add_generation_prompt=True, - tokenize=True, - return_dict=True, - return_tensors="pt", - padding=True, - ) - - -def _load_qwen3_omni( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import torch - from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor - - processor = Qwen3OmniMoeProcessor.from_pretrained( - arguments.model, **_processor_kwargs(arguments) - ) - load_options = _load_kwargs(arguments, torch) - load_options.update( - { - "device_map": str(options.get("device_map", "cuda:0")), - "enable_audio_output": False, - } - ) - model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( - arguments.model, **load_options - ).eval() - conversation = [ - {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT_QWEN3_OMNI}]}, - {"role": "user", "content": [{"type": "text", "text": str(request.get("prompt", ""))}]}, - ] - inputs = _qwen3_omni_chat_inputs(processor, conversation).to(model.device) - thinker_tokens = int(request.get("max_new_tokens", 16)) - - def invoke() -> Mapping[str, Any]: - with torch.inference_mode(): - text_ids = model.generate( - **inputs, - thinker_max_new_tokens=thinker_tokens, - thinker_do_sample=False, - return_audio=False, - ) - input_length = int(inputs["input_ids"].shape[-1]) - generated = text_ids[:, input_length:] - token_ids = [int(token) for token in generated[0].detach().cpu().tolist()] - return { - "text": processor.batch_decode(generated, skip_special_tokens=True)[0].strip(), - "token_ids": token_ids, - "output_tokens": len(token_ids), - } - - return Session(invoke, "transformers") - - -def _load_voicechat( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - """One loaded official offline pipeline, not a prerecorded or live-session oracle.""" - import soundfile as sf - import torch - - request = flatten_config(request) - task = _selected_task(arguments) - if arguments.operation == "speech_dialogue": - if task != "offline_speech_dialogue": - raise ValueError("offline VoiceChat reference cannot qualify live or tool dialogue") - elif arguments.operation == "speak": - if task not in { - "speech_to_speech", - "speech_to_speech_response", - "speech_session", - }: - raise ValueError("VoiceChat speak reference requires a speech-response contract") - else: - raise ValueError("VoiceChat reference supports speak or offline speech_dialogue") - if arguments.precision != "fp32": - raise ValueError("VoiceChat reference currently matches the fp32 native profile") - allowed = { - "audio_path", - "system_prompt", - "seed", - "temperature", - "top_p", - "repetition_penalty", - "presence_penalty", - "tail_frames", - "finish_tail_frames", - } - if unknown := sorted(request.keys() - allowed): - raise ValueError("unsupported offline VoiceChat request fields: " + ", ".join(unknown)) - for key in ("tail_frames", "finish_tail_frames"): - if key in request and ( - isinstance(request[key], bool) or not isinstance(request[key], int) or request[key] != 0 - ): - raise ValueError("offline VoiceChat reference requires zero explicit tail frames") - prompt = request.get("system_prompt", VOICECHAT_SYSTEM_PROMPT) - if not isinstance(prompt, str): - raise ValueError("VoiceChat system_prompt must be text") - # The native empty prompt selects its default, unlike upstream's blank-prompt helper. - if not prompt: - prompt = VOICECHAT_SYSTEM_PROMPT - elif not prompt.strip(): - raise ValueError("whitespace-only prompts do not have matched native/reference semantics") - seed = _request_seed(request, default=0) - if seed < 0: - raise ValueError("VoiceChat reference seed must be nonnegative") - generation = {} - for key in ("temperature", "top_p", "repetition_penalty", "presence_penalty"): - if key in request: - value = request[key] - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - ): - raise ValueError(f"VoiceChat {key} must be finite numeric data") - generation[key] = value - if generation.get("temperature", 0) < 0 or not 0 < generation.get("top_p", 1) <= 1: - raise ValueError("VoiceChat temperature/top_p is outside the supported range") - if generation.get("repetition_penalty", 1) <= 0: - raise ValueError("VoiceChat repetition_penalty must be positive") - - configured_repo = str(options.get("reference_repo", "")) - if not configured_repo: - raise ValueError("nemo-voicechat requires baseline.adapter_options.reference_repo") - reference_repo = Path(_reference_checkout(configured_repo, repository="NeMo Speech")) - utility = reference_repo / "nemo/collections/speechlm2/inference/utils/offline_voicechat.py" - if not utility.is_file(): - raise ValueError("VoiceChat reference checkout lacks the official offline utility") - sys.path.insert(0, str(reference_repo)) - upstream = importlib.import_module( - "nemo.collections.speechlm2.inference.utils.offline_voicechat" - ) - if Path(upstream.__file__).resolve() != utility.resolve(): - raise RuntimeError( - "VoiceChat imported a different NeMo installation; use the requested checkout first" - ) - - checkpoint = _cached_snapshot_path(arguments.model, arguments.revision, "config.json") - if checkpoint is None: - from huggingface_hub import snapshot_download - - checkpoint = Path( - snapshot_download( - repo_id=arguments.model, - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - ) - audio_path = _asset_path(arguments, request, "audio_path") - info = sf.info(audio_path) - # Loading is outside the clock. Reject inputs that would make the official - # WAV helper do otherwise-unmeasured resampling/downmixing there. - if info.samplerate != 16000 or info.channels != 1 or info.frames <= 0: - raise ValueError("matched offline VoiceChat reference requires nonempty 16 kHz mono audio") - model = upstream.build_model(str(checkpoint), device="cuda").float() - if model.source_sample_rate != 16000 or model.target_sample_rate != 22050: - raise ValueError("VoiceChat checkpoint sample rates differ from the matched native profile") - _waveform, signal, lengths = upstream.load_wav_16k_mono(str(audio_path), device="cpu") - - def invoke() -> Mapping[str, Any]: - _seed_all(torch, seed) - # Host-to-device input transfer and prompt preparation belong to the - # public operation, while checkpoint loading and WAV decoding do not. - input_signal, input_lengths = signal.to("cuda"), lengths.to("cuda") - prompt_tokens, prompt_lengths = upstream.encode_system_prompt(model, prompt, device="cuda") - result = upstream.run_offline_inference( - model, - input_signal=input_signal, - input_signal_lens=input_lengths, - prompt_tokens=prompt_tokens, - prompt_token_lens=prompt_lengths, - decode_audio=True, - input_pad_len=0, - **generation, - ) - audio = result.get("audio") - audio_len = result.get("audio_len") - text = result.get("text") - if ( - audio is None - or audio_len is None - or len(audio.shape) != 2 - or audio.shape[0] != 1 - or tuple(audio_len.shape) != (1,) - or not isinstance(text, list) - or len(text) != 1 - or not isinstance(text[0], str) - ): - raise RuntimeError("VoiceChat returned an invalid one-recording audio/text result") - count = audio_len[0].item() - if ( - isinstance(count, bool) - or not isinstance(count, int) - or count < 0 - or count > audio.shape[1] - ): - raise RuntimeError("VoiceChat audio_len does not match the decoded audio") - materialized = audio[0, :count].detach().float().cpu() - if not bool(materialized.isfinite().all().item()): - raise RuntimeError("VoiceChat reference returned nonfinite audio") - return { - "text": text[0], - "audio_samples": count, - "num_samples": count, - "sample_rate": int(model.target_sample_rate), - "channels": 1, - "audio_seconds": count / model.target_sample_rate, - "input_sample_rate": 16000, - "input_channels": 1, - "input_samples": int(info.frames), - "finite": True, - "_audio_f32": materialized.numpy(), - } - - return Session( - invoke, - "nemo-voicechat", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=False, - ) - - -def _load_moge( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import numpy as np - import torch - from PIL import Image - from torch.nn.attention import SDPBackend, sdpa_kernel - - request = flatten_config(request) - if arguments.operation != "geometry" or arguments.precision != "fp32": - raise ValueError("MoGe reference requires geometry with the qualified fp32 profile") - if unknown := sorted(request.keys() - {"image_path", "num_tokens", "fov_x"}): - raise ValueError("unsupported MoGe reference request fields: " + ", ".join(unknown)) - num_tokens = request.get("num_tokens", options.get("num_tokens")) - if isinstance(num_tokens, bool) or not isinstance(num_tokens, int) or num_tokens != 1800: - raise ValueError("MoGe reference requires explicit request/options num_tokens=1800") - if ( - "num_tokens" in request - and "num_tokens" in options - and request["num_tokens"] != options["num_tokens"] - ): - raise ValueError("MoGe request and adapter num_tokens disagree") - inference = { - "num_tokens": num_tokens, - "use_fp16": False, - "force_projection": True, - "apply_mask": True, - } - if "fov_x" in request: - fov = request["fov_x"] - if ( - isinstance(fov, bool) - or not isinstance(fov, (int, float)) - or not math.isfinite(fov) - or not 0 < fov < 180 - ): - raise ValueError("MoGe fov_x must be a finite angle in degrees between zero and 180") - inference["fov_x"] = fov - configured_repo = str(options.get("reference_repo", "")) - if not configured_repo: - raise ValueError("upstream-moge requires baseline.adapter_options.reference_repo") - reference_repo = Path(_reference_checkout(configured_repo, repository="microsoft/MoGe")) - expected_module = reference_repo / "moge/model/v2.py" - if not expected_module.is_file(): - raise ValueError("MoGe reference checkout does not contain the v2 model") - # The upstream checkout supplies its own declared geometry dependencies. - sys.path.insert(0, str(reference_repo)) - upstream = importlib.import_module("moge.model.v2") - if Path(upstream.__file__).resolve() != expected_module.resolve(): - raise RuntimeError("MoGe imported a different installation than the requested checkout") - cached = _cached_snapshot_path(arguments.model, arguments.revision, "model.pt") - if cached is None: - from huggingface_hub import hf_hub_download - - checkpoint_path = Path( - hf_hub_download( - repo_id=arguments.model, - revision=arguments.revision, - filename="model.pt", - local_files_only=arguments.local_files_only, - ) - ) - else: - checkpoint_path = cached if cached.is_file() else cached / "model.pt" - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True, mmap=True) - if set(checkpoint) != {"model_config", "model"}: - raise ValueError("MoGe checkpoint has an unexpected top-level contract") - model = upstream.MoGeModel(**checkpoint["model_config"]) - missing, unexpected = model.load_state_dict(checkpoint["model"], strict=False) - if missing or unexpected: - raise ValueError(f"MoGe state mismatch: missing={missing}, unexpected={unexpected}") - del checkpoint - model.onnx_compatible_mode = True - model.eval().float().to("cuda") - torch.backends.cuda.matmul.allow_tf32 = False - torch.backends.cudnn.allow_tf32 = False - image_path = _asset_path(arguments, request, "image_path") - with Image.open(image_path) as image: - # Match the native caller's cached RGB float HWC input. No resizing or - # model normalization is performed here; upstream infer owns those. - pixels = np.asarray(image.convert("RGB"), dtype=np.float32).copy() / 255.0 - height, width = pixels.shape[:2] - - def invoke() -> Mapping[str, Any]: - tensor = torch.from_numpy(pixels).permute(2, 0, 1).to("cuda") - with sdpa_kernel([SDPBackend.MATH]): - output = model.infer(tensor, **inference) - if set(output) != {"points", "depth", "mask", "intrinsics"}: - raise RuntimeError("MoGe reference must return points, depth, mask and intrinsics") - arrays = {name: value.detach().cpu().numpy() for name, value in output.items()} - return { - "geometry_images": 1, - "geometry_pixels": height * width, - "height": height, - "width": width, - "point_shape": [height, width, 3], - "num_tokens": num_tokens, - "requested_fov_x": request.get("fov_x"), - "units": "meters", - "camera_axes": ["right", "down", "forward"], - "intrinsics_coordinates": "normalized_uv", - "_geometry_arrays": arrays, - } - - return Session( - invoke, - "moge-pytorch", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=False, - ) - - -def _write_geometry_artifacts(summary: dict[str, Any], output: Path) -> None: - """Validate and write complete geometry only after the measurement clock.""" - import numpy as np - - arrays = summary.pop("_geometry_arrays") - height, width = summary["height"], summary["width"] - points, depth, mask, intrinsics = ( - np.asarray(arrays[name]) for name in ("points", "depth", "mask", "intrinsics") - ) - if ( - points.shape != (height, width, 3) - or depth.shape != (height, width) - or mask.shape != (height, width) - or intrinsics.shape != (3, 3) - or not np.isin(mask, (0, 1)).all() - or not np.isfinite(intrinsics).all() - ): - raise RuntimeError("MoGe geometry arrays do not match the original image/calibration") - valid = mask.astype(bool, copy=False) - if ( - not np.isfinite(points[valid]).all() - or not np.isfinite(depth[valid]).all() - or np.any(depth[valid] <= 0) - or not np.isposinf(points[~valid]).all() - or not np.isposinf(depth[~valid]).all() - ): - raise RuntimeError( - "MoGe geometry must preserve valid metric values and masked positive infinity" - ) - prefix = output.with_suffix(".geometry").resolve() - prefix.parent.mkdir(parents=True, exist_ok=True) - for name, values, suffix, dtype in ( - ("points_artifact", points, ".points.f32", " Session: - official_repo = str(options.get("official_repo", "")) - if not official_repo: - raise ValueError("pytorch-personaplex requires baseline.adapter_options.official_repo") - official_repo = _reference_checkout( - official_repo, repository="https://github.com/NVIDIA/personaplex" - ) - import importlib.util - - if importlib.util.find_spec("sphn") is None: - raise RuntimeError("PersonaPlex reference requires the official sphn package") - sys.path[:0] = [str(Path(official_repo) / "moshi"), official_repo] - import torch - from huggingface_hub import hf_hub_download - from moshi.models import LMGen, loaders - from moshi.models.lm import _iterate_audio, encode_from_sphn, load_audio - from moshi.offline import warmup - - device = "cuda" - mimi_weights = hf_hub_download( - arguments.model, - loaders.MIMI_NAME, - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - model_weights = hf_hub_download( - arguments.model, - loaders.MOSHI_NAME, - revision=arguments.revision, - local_files_only=arguments.local_files_only, - ) - mimi = loaders.get_mimi(mimi_weights, device) - other_mimi = loaders.get_mimi(mimi_weights, device) - language_model = loaders.get_moshi_lm(model_weights, device=device).eval() - frame_size = int(mimi.sample_rate / mimi.frame_rate) - generator = LMGen( - language_model, - audio_silence_frame_cnt=0, - sample_rate=mimi.sample_rate, - device=device, - frame_rate=mimi.frame_rate, - use_sampling=False, - temp=0.8, - temp_text=0.7, - top_k=250, - top_k_text=25, - ) - mimi.streaming_forever(1) - other_mimi.streaming_forever(1) - generator.streaming_forever(1) - warmup(mimi, other_mimi, generator, device, frame_size) - source_audio = load_audio(str(_asset_path(arguments, request, "audio_path")), mimi.sample_rate) - max_frames = int(request.get("max_new_tokens", options.get("max_frames", 100))) - - def invoke() -> Mapping[str, Any]: - mimi.reset_streaming() - other_mimi.reset_streaming() - generator.reset_streaming() - generated_frames = 0 - with torch.inference_mode(): - for encoded in encode_from_sphn( - mimi, - _iterate_audio(source_audio, sample_interval_size=frame_size, pad=True), - max_batch=1, - ): - for index in range(encoded.shape[-1]): - tokens = generator.step(encoded[:, :, index : index + 1]) - if tokens is None: - continue - mimi.decode(tokens[:, 1:9]) - other_mimi.decode(tokens[:, 1:9]) - generated_frames += 1 - if generated_frames >= max_frames: - return { - "audio_frames": generated_frames, - "audio_samples": generated_frames * frame_size, - "sample_rate": mimi.sample_rate, - } - return { - "audio_frames": generated_frames, - "audio_samples": generated_frames * frame_size, - "sample_rate": mimi.sample_rate, - } - - return Session( - invoke, - "moshi", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - ) - - -def _load_fast_foundation_stereo( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> Session: - import numpy as np - import torch - from PIL import Image - - from apps.benchmark.performance.baselines.reference_support import ( - configure_official_model_args, - fast_foundation_stereo_model_dir, - ) - - if not torch.cuda.is_available(): - raise RuntimeError("CUDA is required for the Fast Foundation Stereo reference") - max_disp = int(request.get("max_disp", 192)) - valid_iters = int(request.get("valid_iters", 8)) - height = int(request.get("height", 700)) - width = int(request.get("width", 700)) - if (height, width, max_disp, valid_iters) != (700, 700, 192, 8): - raise ValueError( - "Fast Foundation Stereo performance requires 700x700, max_disp=192, and valid_iters=8" - ) - model_dir = options.get("model_dir") - if not isinstance(model_dir, str) or not model_dir: - raise ValueError("Fast Foundation Stereo reference requires adapter_options.model_dir") - model_root = fast_foundation_stereo_model_dir(model_dir) - previous_cwd = Path.cwd() - try: - os.chdir(model_root) - sys.path.insert(0, str(model_root)) - from core.utils.utils import InputPadder - from Utils import AMP_DTYPE - - checkpoint = model_root / "weights/23-36-37/model_best_bp2_serialize.pth" - model = torch.load(checkpoint, map_location="cpu", weights_only=False) - finally: - os.chdir(previous_cwd) - configure_official_model_args( - model, - max_disparity=max_disp, - valid_iters=valid_iters, - ) - model = model.cuda().eval() - - def load_image(key: str) -> np.ndarray: - with Image.open(_asset_path(arguments, request, key)) as image: - pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) - if pixels.shape != (height, width, 3): - raise ValueError(f"{key} must be 700x700 RGB, got {pixels.shape}") - return pixels - - left = load_image("left_image_path") - right = load_image("right_image_path") - - def invoke() -> Mapping[str, Any]: - left_tensor = torch.as_tensor(left).cuda().float()[None].permute(0, 3, 1, 2) - right_tensor = torch.as_tensor(right).cuda().float()[None].permute(0, 3, 1, 2) - padder = InputPadder(left_tensor.shape, divis_by=32, force_square=False) - left_tensor, right_tensor = padder.pad(left_tensor, right_tensor) - with ( - torch.inference_mode(), - torch.amp.autocast("cuda", enabled=True, dtype=AMP_DTYPE), - ): - disparity = model.forward( - left_tensor, - right_tensor, - iters=valid_iters, - test_mode=True, - optimize_build_volume="pytorch1", - ) - output = padder.unpad(disparity.float()).cpu().numpy().reshape(height, width) - output = np.clip(output, 0, None).astype(np.float32, copy=False) - return { - "height": height, - "width": width, - "element_count": int(output.size), - "finite_fraction": float(np.isfinite(output).mean()), - "nonnegative_fraction": float((output >= 0).mean()), - "_disparity_f32": output, - } - - return Session( - invoke, - f"torch-{torch.__version__}", - timing_scope="task-pipeline-call-wall", - input_preparation_included=True, - asset_loading_included=False, - ) - - -LOADERS: dict[ - str, Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session] -] = { - "hf-diffusers": _load_diffusers, - "hf-qwen3-omni": _load_qwen3_omni, - "hf-transformers-asr": _load_asr, - "hf-transformers-embedding": _load_embedding, - "hf-transformers-reranking": _load_reranking, - "hf-transformers-tts": _load_tts, - "hf-transformers-vision": _load_vision, - "hf-transformers-vlm": _load_vlm, - "nemo-asr": _load_asr, - "nemo-tts": _load_tts, - "nemo-voicechat": _load_voicechat, - "upstream-moge": _load_moge, - "pytorch-personaplex": _load_personaplex, - "pytorch-timeseries": _load_timeseries, - "upstream-fast-foundation-stereo": _load_fast_foundation_stereo, -} - - -def _synchronize() -> None: - try: - import torch - - if torch.cuda.is_available(): - torch.cuda.synchronize() - except ImportError: - return - - -def _compile_forward(model: Any) -> dict[str, Any]: - import torch - from torch._dynamo.backends.registry import lookup_backend - - evidence = {"compiled_graph_count": 0} - inductor = lookup_backend("inductor") - - def compile_graph(graph: Any, inputs: Any, **options: Any) -> Any: - compiled = inductor(graph, inputs, **options) - evidence["compiled_graph_count"] += 1 - return compiled - - model.forward = torch.compile( - model.forward, - backend=compile_graph, - fullgraph=False, - dynamic=False, - ) - evidence.update( - { - "api": "torch.compile", - "target": "model.forward", - "backend": "inductor", - "mode": "default", - "fullgraph": False, - "dynamic": False, - "applied": True, - } - ) - return evidence - - -def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: - output: Mapping[str, Any] = {} - for _ in range(warmup): - output = session.invoke() - _synchronize() - compiled_graphs = None - if session.compile_evidence is not None: - compiled_graphs = int(session.compile_evidence["compiled_graph_count"]) - if compiled_graphs < 1: - raise RuntimeError("warmup did not execute a compiled graph") - samples = [] - for _ in range(iterations): - _synchronize() - started = time.perf_counter() - output = session.invoke() - _synchronize() - samples.append((time.perf_counter() - started) * 1000.0) - if ( - compiled_graphs is not None - and int(session.compile_evidence["compiled_graph_count"]) != compiled_graphs - ): - raise RuntimeError("model compilation occurred inside timed samples") - return samples, dict(output) - - -def _run_elf( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - reference_repo = str(options.get("reference_repo", "")) - if not reference_repo: - raise ValueError("upstream-elf requires baseline.adapter_options.reference_repo") - reference_repo = _reference_checkout( - reference_repo, repository="https://github.com/lillian039/ELF" - ) - count = arguments.warmup + arguments.iterations - prompt = str(request.get("prompt", "")) - - def repository_input(name: str) -> str: - value = str(request.get(name, "") or "") - if not value: - return "" - path = Path(value) - model_root = arguments.manifest.resolve().parent.parent - return str(path if path.is_absolute() else (model_root / path).resolve()) - - with tempfile.TemporaryDirectory(prefix="trtmc-perf-elf-") as temporary: - root = Path(temporary) - dataset = root / "dataset.jsonl" - rows = [ - {"id": f"perf_{index:04d}", "input": prompt, "output": ""} for index in range(count) - ] - dataset.write_text( - "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), - encoding="utf-8", - ) - output = root / "predictions.json" - command = [ - sys.executable, - str(REPOSITORY / "apps/benchmark/performance/baselines/elf_reference.py"), - "--reference-repo", - reference_repo, - "--config", - str(options["config"]), - "--checkpoint", - str(options["checkpoint"]), - "--dataset", - str(dataset), - "--output", - str(output), - "--shared-inputs-dir", - str(root / "shared"), - "--generation-mode", - str(request.get("generation_mode", "conditional")), - "--sampling-method", - str(options.get("sampling_method", "ode")), - "--num-steps", - str(request.get("num_steps", 64)), - "--cfg-scale", - str(request.get("cfg_scale", 2.0)), - "--self-cond-cfg-scale", - str(options.get("self_cond_cfg_scale", 1.0)), - "--sde-gamma", - str(options.get("sde_gamma", 0.0)), - "--seed", - str(options.get("seed", 42)), - "--precision", - arguments.precision, - ] - replay_inputs = ( - ("initial_latents_path", "--initial-latents"), - ("sampling_steps_path", "--sampling-steps"), - ("condition_latents_path", "--condition-latents"), - ("condition_mask_path", "--condition-mask"), - ("sde_noises_path", "--sde-noises"), - ) - for request_name, flag in replay_inputs: - value = repository_input(request_name) - if value: - command.extend([flag, value]) - if arguments.local_files_only: - command.append("--local-files-only") - completed = subprocess.run( - command, - cwd=reference_repo, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - f"ELF reference failed with rc={completed.returncode}: {completed.stderr[-2000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - responses = payload.get("responses", []) - if len(responses) != count: - raise RuntimeError(f"ELF reference returned {len(responses)} responses; expected {count}") - samples = [float(row["wall_ms"]) for row in responses[arguments.warmup :]] - summary_row = responses[arguments.warmup] - summary = { - "text": str(summary_row.get("output_text", "")), - "token_ids": summary_row.get("generated_token_ids", []), - "output_tokens": len(summary_row.get("generated_token_ids", [])), - } - return ( - samples, - summary, - "elf-pytorch", - "task-model-call-wall", - False, - False, - ) - - -def _run_lance( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - reference_repo = str(options.get("reference_repo", "")) - if not reference_repo: - raise ValueError("upstream-lance requires baseline.adapter_options.reference_repo") - if arguments.precision != "bf16": - raise ValueError("upstream-lance requires bf16 precision") - with tempfile.TemporaryDirectory(prefix="trtmc-perf-lance-adapter-") as temporary: - output = Path(temporary) / "result.json" - command = [ - sys.executable, - str(REPOSITORY / "apps/benchmark/performance/baselines/lance_reference.py"), - "--reference-repo", - reference_repo, - "--model", - arguments.model, - "--model-subdir", - str(options.get("model_subdir", "Lance_3B")), - "--vit-subdir", - str(options.get("vit_subdir", "Qwen2.5-VL-ViT")), - "--image", - str(_asset_path(arguments, request, "image_path")), - "--prompt", - str(request.get("prompt", "")), - "--instruction", - str( - options.get( - "instruction", - "Look at the image carefully and answer the question.", - ) - ), - "--max-new-tokens", - str(request.get("max_new_tokens", 16)), - "--warmup", - str(arguments.warmup), - "--iterations", - str(arguments.iterations), - "--resolution", - str(options.get("resolution", "image_768res")), - "--height", - str(options.get("height", 768)), - "--width", - str(options.get("width", 768)), - "--output", - str(output), - ] - if arguments.revision: - command.extend(["--revision", arguments.revision]) - if arguments.local_files_only: - command.append("--local-files-only") - completed = subprocess.run( - command, - cwd=reference_repo, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - f"Lance reference failed with rc={completed.returncode}: {completed.stderr[-2000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - samples = [float(value) for value in payload.get("samples_ms", [])] - if len(samples) != arguments.iterations: - raise RuntimeError( - f"Lance reference returned {len(samples)} samples; expected {arguments.iterations}" - ) - return ( - samples, - {"text": str(payload.get("text", "")), "output_tokens": None}, - "lance-pytorch", - "task-pipeline-call-wall", - True, - True, - ) - - -def _run_lerobot_act( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - source_root = str(options.get("source_root", "")) - if not source_root: - raise ValueError("LeRobot ACT reference requires adapter_options.source_root") - source_root = _reference_checkout( - source_root, repository="https://github.com/huggingface/lerobot" - ) - if arguments.operation != "control" or arguments.precision != "fp32": - raise ValueError("LeRobot ACT performance requires control in fp32") - - with tempfile.TemporaryDirectory(prefix="trtmc-perf-lerobot-act-") as temporary: - output = Path(temporary) / "result.json" - command = [ - sys.executable, - str(REPOSITORY / "families/lerobot_act/tests/performance_reference.py"), - "--source-root", - source_root, - "--model", - arguments.model, - "--image", - str(_asset_path(arguments, request, "image_path")), - "--state", - str(_asset_path(arguments, request, "state_path")), - "--warmup", - str(arguments.warmup), - "--iterations", - str(arguments.iterations), - "--output", - str(output), - ] - if arguments.revision: - command.extend(("--revision", arguments.revision)) - if arguments.local_files_only: - command.append("--local-files-only") - completed = subprocess.run( - command, - cwd=REPOSITORY, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - "LeRobot ACT reference failed with " - f"rc={completed.returncode}: {completed.stderr[-2000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - samples = [float(value) for value in payload.get("samples_ms", [])] - if len(samples) != arguments.iterations: - raise RuntimeError( - f"LeRobot ACT reference returned {len(samples)} samples; " - f"expected {arguments.iterations}" - ) - summary = payload.get("output_summary") - if not isinstance(summary, Mapping): - raise RuntimeError("LeRobot ACT reference returned no output summary") - return ( - samples, - dict(summary), - str(payload.get("framework", "lerobot-pytorch")), - "task-pipeline-call-wall", - True, - False, - ) - - -def _run_sana_wm( - arguments: argparse.Namespace, - request: Mapping[str, Any], - options: Mapping[str, Any], -) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: - options = dict(options) - testcase_name = getattr(arguments, "testcase_name", None) - if testcase_name is not None: - manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) - matches = [ - value for value in manifest.get("testcases", []) - if isinstance(value, Mapping) and value.get("name") == testcase_name - ] - if len(matches) != 1: - raise ValueError(f"SANA-WM requires exactly one testcase named {testcase_name!r}") - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay"): - if name in matches[0]: - options.setdefault(name, matches[0][name]) - reference_repo = str(options.get("reference_repo", "")) - if not reference_repo: - raise ValueError("upstream-sana-wm requires baseline.adapter_options.reference_repo") - reference_repo = _reference_checkout( - reference_repo, repository="https://github.com/NVlabs/Sana" - ) - - model_root = arguments.manifest.resolve().parent.parent - - def model_input(value: object) -> Path: - path = Path(str(value)) - return path if path.is_absolute() else (model_root / path).resolve() - - image = model_input(request["image_path"]) - prompt_text = str(request.get("prompt", "")) - if not prompt_text: - raise ValueError("SANA-WM request requires prompt") - action = request.get("action") - if not isinstance(action, str) or not action: - raise ValueError("SANA-WM request requires a non-empty action") - intrinsics = model_input(options.get("intrinsics", "assets/demo_0_intrinsics.npy")) - for label, path in (("image", image), ("intrinsics", intrinsics)): - if not path.is_file(): - raise FileNotFoundError(f"SANA-WM {label} input does not exist: {path}") - - num_frames = ( - request["num_frames"] - if "num_frames" in request - else json.loads(arguments.manifest.read_text(encoding="utf-8"))["video_num_frames"] - ) - with tempfile.TemporaryDirectory(prefix="trtmc-perf-sana-wm-") as temporary: - root = Path(temporary) - output = root / "benchmark.json" - prompt = root / "prompt.txt" - prompt.write_text(prompt_text, encoding="utf-8") - command = [ - sys.executable, - str(REPOSITORY / "apps/benchmark/performance/baselines/sana_wm_reference.py"), - "--reference-repo", - reference_repo, - "--image", - str(image), - "--model-dir", - str(options["model_dir"]), - "--prompt", - str(prompt), - "--action", - action, - "--intrinsics", - str(intrinsics), - "--translation_speed", - str(request["translation_speed"] if "translation_speed" in request else options["translation_speed"]), - "--rotation_speed_deg", - str(request["rotation_speed_deg"] if "rotation_speed_deg" in request else options["rotation_speed_deg"]), - "--num_frames", - str(num_frames), - "--fps", - str(request["fps"] if "fps" in request else options["fps"]), - "--step", - str(request["num_steps"]), - "--cfg_scale", - str(request["cfg_scale"]), - "--flow_shift", - str(request["flow_shift"] if "flow_shift" in request else options["flow_shift"]), - "--seed", - str(request["seed"]), - "--refiner_seed", - str(request["seed"]), - "--warmup", - str(arguments.warmup), - "--iterations", - str(arguments.iterations), - "--output", - str(output), - ] - if request["no_action_overlay"] if "no_action_overlay" in request else options["no_action_overlay"]: - command.append("--no_action_overlay") - completed = subprocess.run( - command, - cwd=reference_repo, - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - "SANA-WM reference failed with " - f"rc={completed.returncode}: {completed.stderr[-4000:]}" - ) - payload = json.loads(output.read_text(encoding="utf-8")) - samples = [float(value) for value in payload.get("samples_ms", [])] - if len(samples) != arguments.iterations: - raise RuntimeError( - f"SANA-WM reference returned {len(samples)} samples; expected {arguments.iterations}" - ) - summary = dict(payload.get("output_summary", {})) - return ( - samples, - summary, - "sana-wm-pytorch", - "task-pipeline-call-wall", - True, - False, - ) - - -def _environment() -> dict[str, Any]: - value: dict[str, Any] = { - "platform": platform.platform(), - "python": platform.python_version(), - "python_executable": sys.executable, - } - try: - import torch - - value["torch"] = torch.__version__ - if torch.cuda.is_available(): - value["gpu"] = torch.cuda.get_device_name(0) - value["cuda"] = torch.version.cuda - except ImportError: - pass - for module_name in ("transformers", "diffusers", "nemo", "chronos", "moshi"): - try: - module = __import__(module_name) - except ImportError: - continue - value[module_name] = str(getattr(module, "__version__", "unknown")) - return value - - -def run(arguments: argparse.Namespace) -> int: - if arguments.warmup < 0 or arguments.iterations <= 0: - raise ValueError("warmup must be non-negative and iterations must be positive") - expected_mode = "pytorch-eager" if arguments.adapter in PYTORCH_ADAPTERS else "hf-eager" - supported_modes = {expected_mode} - if arguments.adapter == "pytorch-timeseries" and arguments.family == "chronos_bolt": - supported_modes.add("torch-compile") - if arguments.mode not in supported_modes: - raise ValueError( - f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}" - ) - request = flatten_config(_json_object(arguments.request_json, "--request-json")) - options = _json_object(arguments.adapter_options_json, "--adapter-options-json") - configured_timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") - fields = ("timing_scope", "input_preparation_included", "asset_loading_included") - expected_timing = None - if configured_timing: - if set(configured_timing) != set(fields): - raise ValueError("--timing-contract-json must declare exactly the three reference timing fields") - declared = timing_contract(runner="task-reference", declared=configured_timing) - expected_timing = {name: declared[name] for name in fields} - load_started = time.perf_counter() - load_seconds: float | None = None - compile_evidence: dict[str, Any] | None = None - if arguments.adapter == "upstream-elf": - samples, output_summary, framework, timing_scope, input_included, asset_included = _run_elf( - arguments, request, options - ) - elif arguments.adapter == "upstream-lance": - ( - samples, - output_summary, - framework, - timing_scope, - input_included, - asset_included, - ) = _run_lance(arguments, request, options) - elif arguments.adapter == "pytorch-lerobot-act": - ( - samples, - output_summary, - framework, - timing_scope, - input_included, - asset_included, - ) = _run_lerobot_act(arguments, request, options) - elif arguments.adapter == "upstream-sana-wm": - ( - samples, - output_summary, - framework, - timing_scope, - input_included, - asset_included, - ) = _run_sana_wm(arguments, request, options) - else: - session = LOADERS[arguments.adapter](arguments, request, options) - compile_evidence = session.compile_evidence - load_seconds = time.perf_counter() - load_started - framework = session.framework - timing_scope = session.timing_scope - input_included = session.input_preparation_included - asset_included = session.asset_loading_included - actual_timing = { - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - } - timing_contract(runner="task-reference", declared=actual_timing) - if expected_timing is not None and actual_timing != expected_timing: - raise RuntimeError( - f"{arguments.family} reference implementation timing drifted: " - f"actual={actual_timing}, declared={expected_timing}" - ) - samples, output_summary = _measure(session, arguments.warmup, arguments.iterations) - if "_geometry_arrays" in output_summary: - _write_geometry_artifacts(output_summary, arguments.output) - disparity = output_summary.pop("_disparity_f32", None) - if disparity is not None: - artifact_path = arguments.output.with_suffix(".disparity.f32").resolve() - artifact_path.parent.mkdir(parents=True, exist_ok=True) - disparity.tofile(artifact_path) - output_summary["disparity_artifact"] = str(artifact_path) - audio = output_summary.pop("_audio_f32", None) - if audio is not None: - import soundfile as sf - artifact_path = arguments.output.with_suffix(".audio.wav").resolve() - artifact_path.parent.mkdir(parents=True, exist_ok=True) - sf.write(artifact_path, audio, output_summary["sample_rate"], subtype="FLOAT") - output_summary["audio_artifact"] = str(artifact_path) - if arguments.adapter in { - "pytorch-lerobot-act", - "upstream-elf", - "upstream-lance", - "upstream-sana-wm", - }: - actual_timing = { - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - } - timing_contract(runner="task-reference", declared=actual_timing) - if expected_timing is not None and actual_timing != expected_timing: - raise RuntimeError( - f"{arguments.family} reference implementation timing drifted: " - f"actual={actual_timing}, declared={expected_timing}" - ) - result = { - "schema_version": "trtmc.perf-baseline/v1", - "status": "completed", - "backend": arguments.adapter, - "adapter": arguments.adapter, - "framework": framework, - "mode": arguments.mode, - "precision": arguments.precision, - "padding": arguments.padding, - "experts_implementation": None, - "compile_scope": "model.forward" if arguments.mode == "torch-compile" else None, - "compile_evidence": compile_evidence, - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - "model_load_included": False, - "model_load_seconds": load_seconds, - "model": arguments.model, - "case_name": arguments.case_name, - "measurement": { - "warmup": arguments.warmup, - "iterations": arguments.iterations, - }, - "measurement_policy": { - "timing_scope": timing_scope, - "input_preparation_included": input_included, - "asset_loading_included": asset_included, - "model_load_excluded": True, - "compile_excluded": True, - "warmup_excluded": True, - "output_materialization_included": True, - }, - "samples_ms": samples, - "metrics": { - "sample_count": len(samples), - "latency_ms": { - "p50": statistics.median(samples), - "min": min(samples), - "max": max(samples), - "mean": statistics.fmean(samples), - }, - }, - "output_summary": output_summary, - "environment": _environment(), - "finished_at": datetime.now(timezone.utc).isoformat(), - } - if compile_evidence is not None: - compile_evidence["warmup_completed"] = True - compile_evidence["timed_callable_uses_compiled_target"] = True - if not all(math.isfinite(float(value)) and float(value) > 0.0 for value in samples): - raise RuntimeError("reference produced an invalid timing sample") - arguments.output.parent.mkdir(parents=True, exist_ok=True) - arguments.output.write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - return 0 - - -def main(argv: Sequence[str] | None = None) -> int: - try: - arguments = build_parser().parse_args(argv) - if arguments.local_files_only: - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ["TRANSFORMERS_OFFLINE"] = "1" - return run(arguments) - except (KeyError, OSError, RuntimeError, TypeError, ValueError) as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp index 344142202c..15b2de6ed3 100644 --- a/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp +++ b/apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace { @@ -130,6 +131,218 @@ int main(int argc, char** argv) { check(summary.at("forecast_elements") == 3, "summary values returned"); check(summary.at("shape") == Json::array({1, 3}), "summary shape returned"); + for (const auto& [task, operation, expected_values, feature_kind] : { + std::tuple{"encoding", "encode", Json::array({5.0, 1.0, 2.0, 3.0}), "token"}, + std::tuple{"embedding", "embed", Json::array({5.0, 4.0}), "pooled"}, + }) { + const auto vector_bundle = + runtime_root / (std::string("benchmark_fake_") + task + ".bundle"); + write_bundle(vector_bundle, task); + Json vector_request = request; + vector_request["case_name"] = std::string("fake-") + task; + vector_request["bundle"] = vector_bundle.string(); + vector_request["operation"] = operation; + vector_request["request"] = {{"prompt", "Hello"}}; + { + std::ofstream request_file(request_path); + request_file << vector_request << '\n'; + } + check(std::system(command.c_str()) == 0, "encoder worker process completed"); + std::ifstream vector_output_file(output_path); + Json vector_result; + vector_output_file >> vector_result; + const auto& vector_summary = vector_result.at("output_summary"); + check(vector_summary.at("values") == expected_values, + "legacy encoder output retains values for Accuracy comparison"); + check(vector_summary.at("feature_kind") == feature_kind, + "legacy encoder output identifies token or pooled semantics"); + std::filesystem::remove(vector_bundle); + } + + const auto feature_image = runtime_root / "benchmark_fake_features.ppm"; + { + std::ofstream image(feature_image, std::ios::binary); + image << "P6\n1 1\n255\n"; + image.write("ABC", 3); + } + const auto feature_bundle = runtime_root / "benchmark_fake_image_features.bundle"; + write_bundle(feature_bundle, "image_features"); + Json feature_request = request; + feature_request["case_name"] = "fake-image-features"; + feature_request["bundle"] = feature_bundle.string(); + feature_request["operation"] = "extract_features"; + feature_request["request"] = {{"image_path", feature_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << feature_request << '\n'; + } + check(std::system(command.c_str()) == 0, "legacy image feature worker completed"); + std::ifstream feature_output_file(output_path); + Json feature_result; + feature_output_file >> feature_result; + const auto& feature_summary = feature_result.at("output_summary"); + check(feature_summary.at("pooler_output") == Json::array({0.25F, 0.75F}), + "legacy image feature output retains pooled values for Accuracy comparison"); + check(feature_summary.at("pooler_output_shape") == Json::array({1, 2}), + "legacy image feature output retains pooled shape"); + std::filesystem::remove(feature_bundle); + std::filesystem::remove(feature_image); + + const auto image_bundle = runtime_root / "benchmark_fake_image_generation.bundle"; + write_bundle(image_bundle, "image_generation"); + Json image_request = request; + image_request["case_name"] = "fake-image-generation"; + image_request["bundle"] = image_bundle.string(); + image_request["operation"] = "generate_image"; + image_request["request"] = {{"prompt", "Hello"}}; + { + std::ofstream request_file(request_path); + request_file << image_request << '\n'; + } + check(std::system(command.c_str()) == 0, + "text-only prompt uses generation when a task also supports editing and batching"); + std::ifstream image_output_file(output_path); + Json image_result; + image_output_file >> image_result; + check(image_result.at("status") == "completed" && + image_result.at("output_summary").at("generated_images") == 1, + "single image generation returns one result"); + const auto image_artifacts = image_result.at("output_summary").at("image_artifacts"); + check(image_artifacts.size() == 1 && + std::filesystem::is_regular_file(runtime_root / image_artifacts.at(0)), + "legacy image generation retains the actual PNG outside timed execution"); + std::filesystem::remove(image_bundle); + + const auto audio_bundle = runtime_root / "benchmark_fake_audio_generation.bundle"; + write_bundle(audio_bundle, "audio_generation"); + Json audio_request = request; + audio_request["case_name"] = "fake-audio-generation"; + audio_request["bundle"] = audio_bundle.string(); + audio_request["operation"] = "generate_audio"; + audio_request["request"] = {{"prompt", "Hello"}}; + { + std::ofstream request_file(request_path); + request_file << audio_request << '\n'; + } + check(std::system(command.c_str()) == 0, "legacy audio worker process completed"); + std::ifstream audio_output_file(output_path); + Json audio_result; + audio_output_file >> audio_result; + const auto audio_artifact = + audio_result.at("output_summary").at("audio_artifact").get(); + check(audio_result.at("status") == "completed" && + std::filesystem::is_regular_file(runtime_root / audio_artifact), + "legacy audio generation retains the actual WAV outside timed execution"); + std::filesystem::remove(audio_bundle); + + const auto legacy_image = runtime_root / "benchmark_legacy_segment.ppm"; + { + std::ofstream image(legacy_image, std::ios::binary); + image << "P6\n2 1\n255\n"; + image.write("ABCDEF", 6); + } + const auto segment_bundle = runtime_root / "benchmark_fake_segmentation.bundle"; + write_bundle(segment_bundle, "segmentation"); + Json segment_request = request; + segment_request["case_name"] = "fake-segmentation"; + segment_request["bundle"] = segment_bundle.string(); + segment_request["operation"] = "segment"; + segment_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << segment_request << '\n'; + } + check(std::system(command.c_str()) == 0, "segmentation worker process completed"); + std::ifstream segment_output_file(output_path); + Json segment_result; + segment_output_file >> segment_result; + check(segment_result.at("output_summary").at("mask") == Json::array({7, 7}), + "legacy segmentation retains labels for Accuracy comparison"); + std::filesystem::remove(segment_bundle); + + const auto detection_bundle = runtime_root / "benchmark_fake_detection.bundle"; + write_bundle(detection_bundle, "object_detection"); + Json detection_request = request; + detection_request["case_name"] = "fake-detection"; + detection_request["bundle"] = detection_bundle.string(); + detection_request["operation"] = "detect"; + detection_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << detection_request << '\n'; + } + check(std::system(command.c_str()) == 0, "detection worker process completed"); + std::ifstream detection_output_file(output_path); + Json detection_result; + detection_output_file >> detection_result; + const auto& detection = detection_result.at("output_summary"); + check(detection.at("boxes") == Json::array({-2, 1, 5, 1, 0, 0, 1, 1}) && + detection.at("scores") == Json::array({0.75, 0.0}) && + detection.at("class_ids") == Json::array({42, 7}) && + detection.at("shape") == Json::array({2, 4}) && + detection.at("coordinates") == "xyxy" && detection.at("units") == "pixels", + "legacy detection retains boxes, scores and classes for Accuracy comparison"); + std::filesystem::remove(detection_bundle); + + const auto prompted_bundle = runtime_root / "benchmark_fake_prompted.bundle"; + write_bundle(prompted_bundle, "prompted_segmentation"); + Json prompted_request = request; + prompted_request["case_name"] = "fake-prompted-segmentation"; + prompted_request["bundle"] = prompted_bundle.string(); + prompted_request["operation"] = "segment_prompted"; + prompted_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << prompted_request << '\n'; + } + check(std::system(command.c_str()) == 0, "prompted segmentation worker process completed"); + std::ifstream prompted_output_file(output_path); + Json prompted_result; + prompted_output_file >> prompted_result; + const auto& prompted = prompted_result.at("output_summary"); + check(prompted.at("masks") == Json::array({1, -1, -2, 2}) && + prompted.at("iou_scores") == Json::array({0.5, 0.75}) && + prompted.at("boxes") == Json::array({{0, 0, 1, 1}, {1, 0, 2, 1}}) && + prompted.at("box_coordinates") == "original_image_pixels_xyxy", + "legacy prompted segmentation retains masks, scores and boxes for comparison"); + std::filesystem::remove(prompted_bundle); + + const auto geometry_bundle = runtime_root / "benchmark_fake_geometry.bundle"; + write_bundle(geometry_bundle, "monocular_geometry"); + Json geometry_request = request; + geometry_request["case_name"] = "fake-monocular-geometry"; + geometry_request["bundle"] = geometry_bundle.string(); + geometry_request["operation"] = "geometry"; + geometry_request["request"] = {{"image_path", legacy_image.string()}}; + { + std::ofstream request_file(request_path); + request_file << geometry_request << '\n'; + } + check(std::system(command.c_str()) == 0, "monocular geometry worker process completed"); + std::ifstream geometry_output_file(output_path); + Json geometry_result; + geometry_output_file >> geometry_result; + const auto& geometry = geometry_result.at("output_summary"); + check(geometry_result.at("task") == "monocular_geometry" && + geometry_result.at("selected_task") == "monocular_geometry" && + geometry.at("geometry_pixels") == 2 && + geometry.at("point_shape") == Json::array({1, 2, 3}) && + geometry.at("valid_pixels") == 1 && + geometry.at("normalized_intrinsics") == + Json::array({{1.0, 0.0, 0.5}, {0.0, 1.0, 0.5}, {0.0, 0.0, 1.0}}) && + std::filesystem::file_size(geometry.at("points_artifact").get()) == + 6 * sizeof(float) && + std::filesystem::file_size(geometry.at("depth_artifact").get()) == + 2 * sizeof(float) && + std::filesystem::file_size( + geometry.at("valid_mask_artifact").get()) == 2, + "legacy monocular geometry retains complete metric artifacts"); + std::filesystem::remove(geometry.at("points_artifact").get()); + std::filesystem::remove(geometry.at("depth_artifact").get()); + std::filesystem::remove(geometry.at("valid_mask_artifact").get()); + std::filesystem::remove(geometry_bundle); + std::filesystem::remove(legacy_image); + Json sdk = request; sdk["runtime_root"] = argv[3]; sdk["operation"] = "generate"; diff --git a/apps/benchmark/trtmc_benchmark/artifact_metrics.py b/apps/benchmark/trtmc_benchmark/artifact_metrics.py new file mode 100644 index 0000000000..87772d5590 --- /dev/null +++ b/apps/benchmark/trtmc_benchmark/artifact_metrics.py @@ -0,0 +1,389 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic numerical metrics over complete task output artifacts.""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any, Mapping + + +def generated_media_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare deterministic generated image/video artifacts without family semantics.""" + import numpy as np + from PIL import Image + + actual, actual_count, actual_indices = _media_artifacts(candidate, "candidate") + expected, expected_count, expected_indices = _media_artifacts(reference, "reference") + if actual_count != expected_count: + raise ValueError("candidate and reference media counts differ") + common_indices = sorted(set(actual_indices) & set(expected_indices)) + required_indices = sorted({0, actual_count // 2, actual_count - 1}) + if not set(required_indices).issubset(common_indices): + raise ValueError("candidate and reference media do not retain the required frames") + actual_by_index = dict(zip(actual_indices, actual, strict=True)) + expected_by_index = dict(zip(expected_indices, expected, strict=True)) + psnr_values = [] + ssim_values = [] + for index in required_indices: + left_path, right_path = actual_by_index[index], expected_by_index[index] + left = np.asarray(Image.open(left_path).convert("RGB"), dtype=np.float64) + right = np.asarray(Image.open(right_path).convert("RGB"), dtype=np.float64) + if left.shape != right.shape: + raise ValueError("candidate and reference media dimensions differ") + delta = left - right + mse = float(np.mean(delta * delta)) + psnr_values.append(100.0 if mse == 0.0 else 20.0 * math.log10(255.0 / math.sqrt(mse))) + left_mean, right_mean = float(left.mean()), float(right.mean()) + left_var, right_var = float(left.var()), float(right.var()) + covariance = float(np.mean((left - left_mean) * (right - right_mean))) + c1, c2 = (0.01 * 255.0) ** 2, (0.03 * 255.0) ** 2 + numerator = (2.0 * left_mean * right_mean + c1) * (2.0 * covariance + c2) + denominator = (left_mean**2 + right_mean**2 + c1) * (left_var + right_var + c2) + ssim_values.append(numerator / denominator) + return { + "media_count": float(actual_count), + "compared_frames": float(len(required_indices)), + "min_psnr": min(psnr_values), + "min_ssim": min(ssim_values), + } + + +def generated_media_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: + minimum_psnr = _finite_threshold(thresholds, "min_psnr") + minimum_ssim = _finite_threshold(thresholds, "min_ssim") + return metrics["min_psnr"] >= minimum_psnr and metrics["min_ssim"] >= minimum_ssim + + +def generated_audio_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare complete generated waveforms using stable signal-level metrics.""" + import numpy as np + + actual, actual_rate = _read_wav( + _artifact_path(candidate.get("audio_artifact"), "candidate audio"), np + ) + expected, expected_rate = _read_wav( + _artifact_path(reference.get("audio_artifact"), "reference audio"), np + ) + if actual_rate <= 0 or expected_rate <= 0 or actual.size == 0 or expected.size == 0: + raise ValueError("candidate and reference audio must be non-empty") + actual = actual.astype(np.float64, copy=False) + expected = expected.astype(np.float64, copy=False) + if not np.isfinite(actual).all() or not np.isfinite(expected).all(): + raise ValueError("candidate and reference audio must be finite") + duration_ratio = (actual.size / actual_rate) / (expected.size / expected_rate) + actual_rms = float(np.sqrt(np.mean(actual * actual))) + expected_rms = float(np.sqrt(np.mean(expected * expected))) + rms_ratio = actual_rms / max(expected_rms, 1.0e-12) + length = min(actual.size, expected.size) + # Compare a bounded aligned prefix; generation duration is gated separately. + fft_size = min(length, 262144) + window = np.hanning(fft_size) + left = np.log1p(np.abs(np.fft.rfft(actual[:fft_size] * window))) + right = np.log1p(np.abs(np.fft.rfft(expected[:fft_size] * window))) + log_spectral_distance = float(np.sqrt(np.mean((left - right) ** 2))) + return { + "duration_ratio": float(duration_ratio), + "rms_ratio": float(rms_ratio), + "log_spectral_distance": log_spectral_distance, + } + + +def generated_audio_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: + return ( + metrics["duration_ratio"] >= _finite_threshold(thresholds, "min_duration_ratio") + and metrics["duration_ratio"] <= _finite_threshold(thresholds, "max_duration_ratio") + and metrics["rms_ratio"] >= _finite_threshold(thresholds, "min_rms_ratio") + and metrics["rms_ratio"] <= _finite_threshold(thresholds, "max_rms_ratio") + and metrics["log_spectral_distance"] + <= _finite_threshold(thresholds, "max_log_spectral_distance") + ) + + +def _media_artifacts(summary: Mapping[str, Any], label: str) -> tuple[list[Path], int, list[int]]: + values = summary.get("frame_artifacts", summary.get("image_artifacts")) + if not isinstance(values, list) or not values: + raise ValueError(f"{label} media artifacts are missing") + paths = [_artifact_path(value, f"{label} media artifact") for value in values] + declared_value = summary.get( + "media_count", summary.get("num_frames", summary.get("generated_frames")) + ) + declared = _integer(declared_value, f"{label} media_count", minimum=1) + configured_indices = summary.get("artifact_indices") + indices = list(range(len(paths))) if configured_indices is None else configured_indices + if ( + not isinstance(indices, list) + or len(indices) != len(paths) + or any(isinstance(value, bool) or not isinstance(value, int) for value in indices) + or sorted(set(indices)) != indices + or indices[0] < 0 + or indices[-1] >= declared + ): + raise ValueError(f"{label} media artifact indices are invalid") + return paths, declared, indices + + +def _read_wav(path: Path, np): + import struct + + payload = path.read_bytes() + if len(payload) < 12 or payload[:4] != b"RIFF" or payload[8:12] != b"WAVE": + raise ValueError("audio artifact must be a RIFF/WAVE file") + chunks = {} + offset = 12 + while offset + 8 <= len(payload): + name = payload[offset : offset + 4] + size = int.from_bytes(payload[offset + 4 : offset + 8], "little") + start, end = offset + 8, offset + 8 + size + if end > len(payload): + raise ValueError("audio artifact contains a truncated WAV chunk") + if name in {b"fmt ", b"data"}: + chunks[name] = payload[start:end] + offset = end + (size & 1) + if len(chunks.get(b"fmt ", b"")) < 16 or b"data" not in chunks: + raise ValueError("audio artifact has no complete format and data chunks") + kind, channels, rate, _, alignment, bits = struct.unpack_from(" float: + value = thresholds.get(name) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ValueError(f"threshold {name} must be finite") + return float(value) + + +def metric_geometry_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare two complete metric-geometry outputs on their common valid grid.""" + import numpy as np + + actual = _metric_geometry(candidate, "candidate", np) + expected = _metric_geometry(reference, "reference", np) + actual_points, actual_depth, actual_mask, actual_intrinsics = actual + ref_points, ref_depth, ref_mask, ref_intrinsics = expected + if actual_depth.shape != ref_depth.shape: + raise ValueError("candidate and reference geometry grids differ") + actual_valid = actual_mask.astype(bool, copy=False) + ref_valid = ref_mask.astype(bool, copy=False) + common = actual_valid & ref_valid + union = actual_valid | ref_valid + if not np.any(common): + raise ValueError("candidate and reference geometry have no common valid pixels") + + actual_depth_valid = actual_depth[common].astype(np.float64) + ref_depth_valid = ref_depth[common].astype(np.float64) + depth_delta = actual_depth_valid - ref_depth_valid + actual_points_valid = actual_points[common].astype(np.float64) + ref_points_valid = ref_points[common].astype(np.float64) + point_delta = actual_points_valid - ref_points_valid + cosine_denominator = np.maximum( + np.linalg.norm(actual_points_valid, axis=-1) * np.linalg.norm(ref_points_valid, axis=-1), + 1.0e-12, + ) + ref_intrinsics64 = ref_intrinsics.astype(np.float64) + intrinsics_delta = actual_intrinsics.astype(np.float64) - ref_intrinsics64 + nonzero = np.abs(ref_intrinsics64) > 1.0e-12 + metrics = { + "mask_iou": float(common.sum() / union.sum()), + "depth_absrel_mean": float( + np.mean(np.abs(depth_delta) / np.maximum(np.abs(ref_depth_valid), 1.0e-12)) + ), + "depth_rel_l2": float( + np.linalg.norm(depth_delta) / max(float(np.linalg.norm(ref_depth_valid)), 1.0e-12) + ), + "points_rel_l2": float( + np.linalg.norm(point_delta) / max(float(np.linalg.norm(ref_points_valid)), 1.0e-12) + ), + "points_cosine": float( + np.mean(np.sum(actual_points_valid * ref_points_valid, axis=-1) / cosine_denominator) + ), + "intrinsics_max_relative_error": float( + np.max(np.abs(intrinsics_delta[nonzero]) / np.abs(ref_intrinsics64[nonzero])) + ), + "point_depth_consistency": float( + np.max(np.abs(actual_points[..., 2][actual_valid] - actual_depth[actual_valid])) + ), + } + if not all(math.isfinite(value) for value in metrics.values()): + raise ValueError("metric-geometry comparison produced non-finite metrics") + return metrics + + +def metric_geometry_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: + """Apply the public metric-geometry threshold vocabulary.""" + required = { + "mask_iou", + "depth_absrel_mean", + "depth_rel_l2", + "points_rel_l2", + "points_cosine", + "intrinsics_max_relative_error", + "point_depth_consistency", + } + if set(metrics) != required or not required.issubset(thresholds): + raise ValueError("metric-geometry thresholds are incomplete") + minimums = {"mask_iou", "points_cosine"} + for name in required: + threshold = thresholds[name] + if isinstance(threshold, bool) or not isinstance(threshold, (int, float)): + raise ValueError(f"metric-geometry threshold {name} must be numeric") + if not math.isfinite(float(threshold)): + raise ValueError(f"metric-geometry threshold {name} must be finite") + if name in minimums: + if metrics[name] < float(threshold): + return False + elif metrics[name] > float(threshold): + return False + return True + + +def robot_action_metrics( + candidate: Mapping[str, Any], reference: Mapping[str, Any] +) -> dict[str, float]: + """Compare every value in two robot action chunks.""" + actual = _robot_actions(candidate, "candidate") + expected = _robot_actions(reference, "reference") + if len(actual) != len(expected): + raise ValueError("candidate and reference action chunks differ in size") + deltas = [float(left) - float(right) for left, right in zip(actual, expected, strict=True)] + metrics = { + "action_max_abs_error": max(abs(value) for value in deltas), + "action_mean_abs_error": math.fsum(abs(value) for value in deltas) / len(deltas), + "action_rmse": math.sqrt(math.fsum(value * value for value in deltas) / len(deltas)), + } + if not all(math.isfinite(value) for value in metrics.values()): + raise ValueError("robot action comparison produced non-finite metrics") + return metrics + + +def robot_action_passes(metrics: Mapping[str, float], thresholds: Mapping[str, Any]) -> bool: + """Apply the public robot-action threshold vocabulary.""" + required = { + "action_max_abs_error", + "action_mean_abs_error", + "action_rmse", + } + if set(metrics) != required or not required.issubset(thresholds): + raise ValueError("robot action thresholds are incomplete") + for name in required: + threshold = thresholds[name] + if isinstance(threshold, bool) or not isinstance(threshold, (int, float)): + raise ValueError(f"robot action threshold {name} must be numeric") + if not math.isfinite(float(threshold)) or float(threshold) < 0.0: + raise ValueError(f"robot action threshold {name} must be finite and non-negative") + if metrics[name] > float(threshold): + return False + return True + + +def _robot_actions(summary: Mapping[str, Any], label: str) -> list[float]: + steps = _integer(summary.get("action_steps"), f"{label} action_steps", minimum=1) + dimension = _integer(summary.get("action_dim"), f"{label} action_dim", minimum=1) + count = _integer(summary.get("action_values"), f"{label} action_values", minimum=1) + values = summary.get("actions") + if count != steps * dimension: + raise ValueError(f"{label} action shape does not match its value count") + if not isinstance(values, list) or len(values) != count: + raise ValueError(f"{label} action values are missing or incomplete") + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + for value in values + ): + raise ValueError(f"{label} action values must be finite numbers") + if label == "candidate" and summary.get("within_training_bounds") is not True: + raise ValueError("candidate action values are outside the training bounds") + if label == "reference" and summary.get("finite") is not True: + raise ValueError("reference action values are not declared finite") + return [float(value) for value in values] + + +def _metric_geometry(summary: Mapping[str, Any], label: str, np): + height = _integer(summary.get("height"), f"{label} height", minimum=1) + width = _integer(summary.get("width"), f"{label} width", minimum=1) + pixels = height * width + if _integer(summary.get("geometry_images"), f"{label} geometry_images", minimum=1) != 1: + raise ValueError(f"{label} geometry must contain one image") + if _integer(summary.get("geometry_pixels"), f"{label} geometry_pixels", minimum=1) != pixels: + raise ValueError(f"{label} geometry pixel count differs from its grid") + if summary.get("point_shape") != [height, width, 3]: + raise ValueError(f"{label} point shape differs from [H,W,3]") + if ( + summary.get("units") != "meters" + or summary.get("camera_axes") != ["right", "down", "forward"] + or summary.get("intrinsics_coordinates") != "normalized_uv" + ): + raise ValueError(f"{label} geometry coordinate contract is invalid") + points = _float_artifact(summary, "points_artifact", (height, width, 3), label, np) + depth = _float_artifact(summary, "depth_artifact", (height, width), label, np) + mask = _mask_artifact(summary, pixels, (height, width), label, np) + intrinsics = np.asarray(summary.get("normalized_intrinsics"), dtype=np.float32) + if intrinsics.shape != (3, 3) or not np.isfinite(intrinsics).all(): + raise ValueError(f"{label} normalized intrinsics are invalid") + valid = mask.astype(bool, copy=False) + if not np.any(valid): + raise ValueError(f"{label} geometry has no valid pixels") + if not np.isfinite(points[valid]).all(): + raise ValueError(f"{label} valid points are not finite") + if not np.isfinite(depth[valid]).all() or np.any(depth[valid] <= 0.0): + raise ValueError(f"{label} valid depth is not finite and positive") + declared_valid = _integer(summary.get("valid_pixels"), f"{label} valid_pixels") + if declared_valid != int(valid.sum()): + raise ValueError(f"{label} valid pixel count differs from its mask") + return points, depth, mask, intrinsics + + +def _float_artifact(summary, name, shape, label, np): + path = _artifact_path(summary.get(name), f"{label} {name}") + values = np.fromfile(path, dtype=" Path: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} is missing") + path = Path(value) + if not path.is_file(): + raise ValueError(f"{label} is unavailable") + return path + + +def _integer(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{label} must be an integer >= {minimum}") + return value diff --git a/apps/benchmark/trtmc_benchmark/task_adapters.py b/apps/benchmark/trtmc_benchmark/task_adapters.py index 22600ffc9d..3283d64548 100644 --- a/apps/benchmark/trtmc_benchmark/task_adapters.py +++ b/apps/benchmark/trtmc_benchmark/task_adapters.py @@ -58,6 +58,7 @@ class CaseResolution: "image_text_to_instance_masks": ("segment_prompted", 10, 100), "stereo_images_to_disparity": ("disparity", 3, 100), "image_to_metric_geometry": ("geometry", 3, 100), + "monocular_geometry": ("geometry", 3, 100), "text_to_pooled_features": ("encode", 50, 500), "text_to_head_scores": ("head_scores", 50, 500), "text_to_token_features": ("encode", 50, 500), @@ -312,7 +313,7 @@ def _request(task: str, case: Mapping[str, Any], root: Path) -> dict[str, Any]: return _text_controls(case, include_defaults=False) if task in _LATENT_TEXT | _LATENT_STEPS: return _latent_request(task, case, root) - if task == "image_to_metric_geometry": + if task in {"image_to_metric_geometry", "monocular_geometry"}: request = {"image_path": _semantic_asset(case, root, "image_path", "image", "test_image")} _copy_explicit(request, case, "fov_x") return request @@ -408,8 +409,11 @@ def _request(task: str, case: Mapping[str, Any], root: Path) -> dict[str, Any]: return {"prompt": _prompt(case, root), "batch_size": 1} if task == "reranking": inputs = _inputs(case) - query = inputs.get("query", inputs.get("prompt", case.get("prompt"))) - documents = inputs.get("documents") + query = inputs.get( + "query", + inputs.get("prompt", case.get("query", case.get("prompt"))), + ) + documents = inputs.get("documents", case.get("documents")) if not isinstance(query, str) or not query: raise BenchmarkError("reranking testcase requires inputs.prompt or inputs.query") if not isinstance(documents, list) or not documents: @@ -988,7 +992,7 @@ def _prompt(case: Mapping[str, Any], root: Path) -> str: def _image_path(case: Mapping[str, Any], root: Path) -> str: - return _required_asset(case, ("test_image", "image"), root, "image") + return _required_asset(case, ("image_path", "test_image", "image"), root, "image") def _audio_path(case: Mapping[str, Any], root: Path) -> str: diff --git a/apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py b/apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py new file mode 100644 index 0000000000..31cc7cbda4 --- /dev/null +++ b/apps/benchmark/trtmc_benchmark/tests/test_artifact_metrics.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +import wave + +import numpy as np +from PIL import Image + +from trtmc_benchmark.artifact_metrics import ( + generated_audio_metrics, + generated_audio_passes, + generated_media_metrics, + generated_media_passes, +) + + +def test_generated_media_compares_retained_video_frames(tmp_path: Path) -> None: + left = tmp_path / "left.png" + right = tmp_path / "right.png" + pixels = np.full((8, 8, 3), 96, dtype=np.uint8) + Image.fromarray(pixels).save(left) + Image.fromarray(pixels).save(right) + candidate = { + "media_count": 5, + "artifact_indices": [0, 2, 4], + "frame_artifacts": [str(left), str(left), str(left)], + } + reference = { + "media_count": 5, + "artifact_indices": [0, 2, 4], + "frame_artifacts": [str(right), str(right), str(right)], + } + + metrics = generated_media_metrics(candidate, reference) + + assert metrics["media_count"] == 5 + assert metrics["min_psnr"] == 100.0 + assert generated_media_passes(metrics, {"min_psnr": 5.0, "min_ssim": 0.1}) + + +def test_generated_audio_compares_complete_waveforms(tmp_path: Path) -> None: + sample_rate = 16_000 + time = np.arange(sample_rate, dtype=np.float32) / sample_rate + samples = np.sin(2.0 * np.pi * 440.0 * time).astype(np.float32) + candidate = tmp_path / "candidate.wav" + reference = tmp_path / "reference.wav" + pcm = np.rint(samples * 32767.0).astype(" None: + result = resolve_task_case("monocular_geometry", {"image": "image.ppm"}, sdk_assets) + + assert result.operation == "geometry" + assert result.request == {"image_path": str(sdk_assets / "image.ppm")} + + def test_image_only_detection_has_no_text_or_postprocessing_defaults(sdk_assets: Path) -> None: result = resolve_task_case("image_to_boxes", {"image": "image.ppm", "config": {"score": 0.0}}, sdk_assets) assert result.operation == "detect" @@ -350,6 +357,28 @@ def test_image_only_detection_has_no_text_or_postprocessing_defaults(sdk_assets: assert metrics["images_per_s"] == 50.0 and metrics["detections_per_s"] == 0.0 +def test_legacy_image_task_accepts_canonical_image_path(sdk_assets: Path) -> None: + result = resolve_task_case("object_detection", {"image_path": "image.ppm"}, sdk_assets) + + assert result.request == { + "image_path": str(sdk_assets / "image.ppm"), + "batch_size": 1, + } + + +def test_legacy_reranking_accepts_canonical_query_and_documents(sdk_assets: Path) -> None: + result = resolve_task_case( + "reranking", + {"query": "question", "documents": ["second", "first"]}, + sdk_assets, + ) + + assert result.request == { + "query": "question", + "documents": ["second", "first"], + } + + def test_structure_input_keeps_document_bytes_and_source_path_opaque(tmp_path: Path) -> None: document = tmp_path / "prepared.bytes" document.write_bytes(b"B2RQ\0\x7f") diff --git a/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py b/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py index 964618daa6..f91fcab68b 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py +++ b/apps/benchmark/trtmc_benchmark/tests/test_reporting_and_cache.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - """Behavioral controls for report evidence and managed bundle reuse.""" from __future__ import annotations @@ -761,50 +760,3 @@ def test_report_image_artifacts_obey_case_boundary_and_do_not_invent_time(tmp_pa assert "outside.png" not in document and "link.png" not in document assert "Output images: 4 artifact(s) unavailable" in document assert "Frame 0" in document and "Frame 0 ·" not in document - - -@pytest.mark.parametrize( - "schema, row_key, reference_key", [("v1", "cases", "baseline"), ("v2", "rows", "reference")] -) -def test_performance_report_preserves_historical_measurements( - tmp_path, schema, row_key, reference_key -): - from tools import perf_matrix - - scope = "model_call_wall" if schema == "v1" else "public_task_call_wall" - results = { - "schema_version": "trtmc.perf-matrix/" + schema, - "status": "completed", - "selected_entry_ids": ["example.generate"], - row_key: [ - { - "id": "example.generate", - "model": "example", - "operation": "generate", - "status": "green", - "candidate": {"samples_ms": [8.0, 10.0, 12.0], "timing_scope": scope}, - reference_key: { - "samples_ms": [12.0, 14.0, 16.0], - "measurement_policy": {"timing_scope": "public_operation_call_wall"}, - }, - "commands": {"candidate": {"argv": ["trtmc-bench", "run", "--model", "example"]}}, - } - ], - } - original = json.dumps(results) - (tmp_path / "results.json").write_text(original) - assert perf_matrix.main(["report", str(tmp_path)]) == 0 - report = json.loads((tmp_path / "report.json").read_text()) - document = (tmp_path / "report.html").read_text() - assert report["source_schema_version"] == results["schema_version"] - assert report["rows"][0][reference_key] == results[row_key][0][reference_key] - assert "p50: 10.000 ms" in document - assert "p95: 11.800 ms" in document - assert scope in document - assert "public_operation_call_wall" in document - assert "trtmc-bench" in document - assert 'id="filter"' in document - assert (tmp_path / "results.json").read_text() == original - if schema == "v1": - with pytest.raises(perf_matrix.PerfMatrixError, match="unsupported results schema"): - perf_matrix._load_results(tmp_path) diff --git a/core/builder/tensorrt_model_connect/model_support.py b/core/builder/tensorrt_model_connect/model_support.py index 5a7097202d..04fd18c307 100644 --- a/core/builder/tensorrt_model_connect/model_support.py +++ b/core/builder/tensorrt_model_connect/model_support.py @@ -161,7 +161,13 @@ def resolve_model(model: str, revision: str | None = None) -> Path: raise ValueError(f"model path is not a directory: {local}") from huggingface_hub import snapshot_download - return Path(snapshot_download(repo_id=model, revision=revision)) + return Path( + snapshot_download( + repo_id=model, + revision=revision, + ignore_patterns=["*flax_model*", "*tf_model*"], + ) + ) def _family_directories() -> list[Path]: diff --git a/core/builder/tests/test_build_cli.py b/core/builder/tests/test_build_cli.py index 33dc8c9703..8086a1524b 100644 --- a/core/builder/tests/test_build_cli.py +++ b/core/builder/tests/test_build_cli.py @@ -338,7 +338,13 @@ def snapshot_download(**kwargs): ) assert build_cli._resolve_model("openai-community/gpt2", "revision-1") == tmp_path - assert calls == [{"repo_id": "openai-community/gpt2", "revision": "revision-1"}] + assert calls == [ + { + "repo_id": "openai-community/gpt2", + "revision": "revision-1", + "ignore_patterns": ["*flax_model*", "*tf_model*"], + } + ] def test_build_command_rejects_a_task_the_family_does_not_own(monkeypatch, tmp_path: Path) -> None: diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 289084c219..adf4bd3b42 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -42,6 +42,147 @@ class FakeForecast final : public trtmc::ITimeSeriesForecast { std::uint64_t kv_cache_size_bytes_; }; +class FakeEncoding final : public trtmc::IEncoding { + public: + trtmc::EmbeddingResult encode(const std::string& text) override { + return {{static_cast(text.size()), 1.0F, 2.0F, 3.0F}, 2}; + } +}; + +class FakeEmbedding final : public trtmc::IEmbedding { + public: + trtmc::EmbeddingResult embed(const std::string& text) override { + return {{static_cast(text.size()), 4.0F}, 2}; + } +}; + +class FakeImageFeatures final : public trtmc::IImageFeatureExtractor { + public: + trtmc::ImageFeaturesResult extract_image_features(const float*, std::int32_t, + std::int32_t) override { + return {{1.0F, 2.0F, 3.0F, 4.0F}, {1, 2, 2}, {0.25F, 0.75F}, {1, 2}}; + } +}; + +class FakeImageGeneration final : public trtmc::IImageGeneration, + public trtmc::IImageEditing, + public trtmc::IImageBatchGeneration { + public: + const char* task() const noexcept override { return trtmc::IImageGeneration::kTask; } + + trtmc::ImageResult generate_image(const std::string&, + const trtmc::ImageGenerationConfig&) override { + return {{0.25F, 0.5F, 0.75F}, 1, 1, 3, 1}; + } + + trtmc::ImageResult generate_image(const std::string&, const float*, std::int32_t, std::int32_t, + const trtmc::ImageGenerationConfig&) override { + return {{0.75F, 0.5F, 0.25F}, 1, 1, 3, 1}; + } + + std::vector + generate_image_batch(const std::vector& prompts, const std::vector&, + const trtmc::ImageGenerationConfig& config) override { + std::vector results; + for (const auto& prompt : prompts) + results.push_back(generate_image(prompt, config)); + return results; + } +}; + +class FakeAudioGeneration final : public trtmc::IAudioGeneration { + public: + trtmc::AudioResult generate_audio(const std::string&, + const trtmc::AudioGenerationConfig&) override { + return {{0.25F, -0.5F, 0.75F, -1.0F}, 4, 8000}; + } +}; + +class FakeSegmentation final : public trtmc::ISegmentation { + public: + trtmc::SegmentResult segment(const float*, std::int32_t height, std::int32_t width) override { + return {std::vector(static_cast(height) * width, 7), height, + width}; + } +}; + +class FakeObjectDetection final : public trtmc::IObjectDetection { + public: + trtmc::ObjectDetectionResult detect(const float*, std::int32_t height, + std::int32_t width) override { + trtmc::ObjectDetectionResult result; + result.boxes = {{-2.0F, 1.0F, 5.0F, 1.0F, 0.75F, 42}, {0.0F, 0.0F, 1.0F, 1.0F, 0.0F, 7}}; + result.image_height = height; + result.image_width = width; + return result; + } +}; + +class FakeMonocularGeometry final : public trtmc::IMonocularGeometry { + public: + trtmc::GeometryResult estimate_geometry(const float*, std::int32_t height, + std::int32_t width) override { + trtmc::GeometryResult result; + result.points = {1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F}; + result.depth = {3.0F, 6.0F}; + result.mask = {1, 0}; + result.intrinsics = {1.0F, 0.0F, 0.5F, 0.0F, 1.0F, 0.5F, 0.0F, 0.0F, 1.0F}; + result.height = height; + result.width = width; + return result; + } +}; + +class FakePointPromptedSegmentation final : public trtmc::IPointPromptedSegmentation { + public: + trtmc::PromptedSegmentationResult segment_prompted(const float*, std::int32_t height, + std::int32_t width, float, float, + bool) override { + trtmc::PromptedSegmentationResult result; + result.masks = {1.0F, -1.0F, -2.0F, 2.0F}; + result.iou_scores = {0.5F, 0.75F}; + result.boxes = {0.0F, 0.0F, 1.0F, 1.0F, 1.0F, 0.0F, 2.0F, 1.0F}; + result.num_masks = 2; + result.height = height; + result.width = width; + return result; + } +}; + +trtmc::ITask* create_fake_perception_task(const trtmc::FamilyContext& context) { + if (context.reader.info().task == trtmc::ISegmentation::kTask) + return new FakeSegmentation(); + if (context.reader.info().task == trtmc::IObjectDetection::kTask) + return new FakeObjectDetection(); + if (context.reader.info().task == trtmc::IMonocularGeometry::kTask) + return new FakeMonocularGeometry(); + if (context.reader.info().task == trtmc::IPointPromptedSegmentation::kTask) + return new FakePointPromptedSegmentation(); + return nullptr; +} + +trtmc::ITask* create_fake_task(const trtmc::FamilyContext& context) { + if (auto* task = create_fake_perception_task(context)) + return task; + if (context.reader.info().task == trtmc::IEncoding::kTask) + return new FakeEncoding(); + if (context.reader.info().task == trtmc::IEmbedding::kTask) + return new FakeEmbedding(); + if (context.reader.info().task == trtmc::IImageFeatureExtractor::kTask) + return new FakeImageFeatures(); + if (context.reader.info().task == trtmc::IImageGeneration::kTask) + return new FakeImageGeneration(); + if (context.reader.info().task == trtmc::IAudioGeneration::kTask) + return new FakeAudioGeneration(); + if (context.reader.info().task == trtmc::ITimeSeriesForecast::kTask) + return new FakeForecast(context.backend, context.kv_cache_size_bytes); + if (context.reader.info().task == trtmc::ITextGeneration::kTask) { + // Deliberately violate the factory contract for the loader's mismatch test. + return new FakeForecast(context.backend, context.kv_cache_size_bytes); + } + throw std::runtime_error("unsupported fake task"); +} + } // namespace extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { @@ -55,5 +196,5 @@ extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context const auto plan = context.reader.read_section("engine.plan"); if (std::string(plan.begin(), plan.end()) != "PLAN") throw std::runtime_error("unexpected engine plan"); - return new FakeForecast(context.backend, context.kv_cache_size_bytes); + return create_fake_task(context); } diff --git a/core/runtime/tests/test_family_loader.cpp b/core/runtime/tests/test_family_loader.cpp index f7bf734f50..73a6bba726 100644 --- a/core/runtime/tests/test_family_loader.cpp +++ b/core/runtime/tests/test_family_loader.cpp @@ -154,7 +154,7 @@ int main(int argc, char** argv) { check(load_throws(unsafe_bundle, runtime_root.string()), "unsafe family id rejected"); const auto mismatch_bundle = runtime_root / "mismatch.bundle"; - write_bundle(mismatch_bundle, "fake", "embedding"); + write_bundle(mismatch_bundle, "fake", "text_generation"); check(load_throws(mismatch_bundle, runtime_root.string()), "factory task must exactly match bundle task"); diff --git a/families/albert/tests/benchmark/albert-base.yaml b/families/albert/tests/benchmark/albert-base.yaml new file mode 100644 index 0000000000..b8045f41f9 --- /dev/null +++ b/families/albert/tests/benchmark/albert-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: albert-base +candidate: + family: albert + checkpoint: albert/albert-base-v2 + revision: 8e2f239c5f8a2c0f253781ca60135db913e5c80c + task: encoding + precision: fp16 + bundle: albert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.99 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/bark/tests/benchmark/bark-large.yaml b/families/bark/tests/benchmark/bark-large.yaml new file mode 100644 index 0000000000..c9f726b3f2 --- /dev/null +++ b/families/bark/tests/benchmark/bark-large.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: bark-large +candidate: {family: bark, checkpoint: suno/bark, revision: 70a8a7d34168586dc5d028fa9666aceade177992, task: audio_generation, precision: fp32, bundle: bark-large.bundle, build: {max_sequence_length: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-audio-parity + benchmark: task_output_parity + operation: generate_audio + request: &request {prompt: "Hello, this is a test of the text to speech system.", max_new_tokens: 768, seed: 42} + reference: &reference {adapter: hf-transformers-tts, mode: hf-eager, precision: fp32, output_contract: audio-artifact-parity} + gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} +performance: + - {name: generate-audio, benchmark: audio_generation_model_performance, operation: generate_audio, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_transformers, output_contract: audio-shape}} diff --git a/families/bark/tests/benchmark/bark-small.yaml b/families/bark/tests/benchmark/bark-small.yaml new file mode 100644 index 0000000000..386052b15b --- /dev/null +++ b/families/bark/tests/benchmark/bark-small.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: bark-small +candidate: {family: bark, checkpoint: suno/bark-small, revision: 1dbd7a128513b8ae4a4e2130fed57b7ac9da5bcd, task: audio_generation, precision: fp16, bundle: bark-small.bundle, build: {max_sequence_length: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-audio-parity + benchmark: task_output_parity + operation: generate_audio + request: &request {prompt: "Hello, this is a test of the audio generation system.", max_new_tokens: 768, seed: 0} + reference: &reference {adapter: hf-transformers-tts, mode: hf-eager, precision: fp16, output_contract: audio-artifact-parity} + gate: &gate {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} +performance: + - {name: generate-audio, benchmark: audio_generation_model_performance, operation: generate_audio, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_transformers, output_contract: audio-shape}} diff --git a/families/bart/tests/benchmark/bart-base.yaml b/families/bart/tests/benchmark/bart-base.yaml new file mode 100644 index 0000000000..a4d35b3f14 --- /dev/null +++ b/families/bart/tests/benchmark/bart-base.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: bart-base +candidate: + family: bart + checkpoint: facebook/bart-base + revision: aadd2ab0ae0c8268c7c9693540e9904811f36177 + task: text_generation + precision: fp16 + bundle: bart-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: wmt14-en-de + benchmark: wmt14_en_de_translation + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp32 + task: seq2seq-lm + output_token_policy: strip-start-and-eos + request: &greedy-generation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start-and-eos + precision: fp32 + output_contract: exact-token-ids diff --git a/families/bert/tests/benchmark/all-minilm-l6-v2.yaml b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml new file mode 100644 index 0000000000..3d7ec772c8 --- /dev/null +++ b/families/bert/tests/benchmark/all-minilm-l6-v2.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: all-minilm-l6-v2 +candidate: + family: bert + checkpoint: sentence-transformers/all-MiniLM-L6-v2 + revision: 1110a243fdf4706b3f48f1d95db1a4f5529b4d41 + task: embedding + precision: fp16 + bundle: all-minilm-l6-v2.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: embed + benchmark: embedding_performance + operation: embed + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-embedding + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + reference_backend: hf_transformers + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/bert-base-uncased.yaml b/families/bert/tests/benchmark/bert-base-uncased.yaml new file mode 100644 index 0000000000..e99f8a34ec --- /dev/null +++ b/families/bert/tests/benchmark/bert-base-uncased.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: bert-base-uncased +candidate: + family: bert + checkpoint: google-bert/bert-base-uncased + revision: 86b5e0934494bd15c9632b12f734a8a67f723594 + task: encoding + precision: fp16 + bundle: bert-base-uncased.bundle + build: + max_sequence_length: 128 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/bge-small-en-v1.5.yaml b/families/bert/tests/benchmark/bge-small-en-v1.5.yaml new file mode 100644 index 0000000000..cdf998a260 --- /dev/null +++ b/families/bert/tests/benchmark/bge-small-en-v1.5.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: bge-small-en-v1.5 +candidate: + family: bert + checkpoint: BAAI/bge-small-en-v1.5 + revision: 5c38ec7c405ec4b44b94cc5a9bb96e735b38267a + task: encoding + precision: fp16 + bundle: bge-small-en-v1.5.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/multilingual-e5-small.yaml b/families/bert/tests/benchmark/multilingual-e5-small.yaml new file mode 100644 index 0000000000..649a3154ce --- /dev/null +++ b/families/bert/tests/benchmark/multilingual-e5-small.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: multilingual-e5-small +candidate: + family: bert + checkpoint: intfloat/multilingual-e5-small + revision: 614241f622f53c4eeff9890bdc4f31cfecc418b3 + task: embedding + precision: fp16 + bundle: multilingual-e5-small.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: embed + benchmark: embedding_performance + operation: embed + request: + prompt: "query: What is artificial intelligence?" + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-embedding + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + reference_backend: hf_transformers + precision: fp32 + output_contract: embedding-shape diff --git a/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml b/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml new file mode 100644 index 0000000000..d5010f9bf2 --- /dev/null +++ b/families/bert/tests/benchmark/paraphrase-multilingual-minilm-l12-v2.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: paraphrase-multilingual-minilm-l12-v2 +candidate: + family: bert + checkpoint: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + revision: e8f8c211226b894fcb81acc59f3b34ba3efd5f42 + task: encoding + precision: fp16 + bundle: paraphrase-multilingual-minilm-l12-v2.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/canary/tests/benchmark/canary-1b-v2.yaml b/families/canary/tests/benchmark/canary-1b-v2.yaml new file mode 100644 index 0000000000..07bcd38999 --- /dev/null +++ b/families/canary/tests/benchmark/canary-1b-v2.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: canary-1b-v2 +candidate: + family: canary + checkpoint: nvidia/canary-1b-v2 + revision: d455706339a6b32e1aa40f82c713a482a0c938e2 + task: transcription + precision: fp16 + bundle: canary-1b-v2.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + language: en + max_new_tokens: 50 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.3 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_nemo_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: nemo_reference + precision: fp32 + output_contract: transcription-text diff --git a/families/canary/tests/benchmark/reference.py b/families/canary/tests/benchmark/reference.py new file mode 100644 index 0000000000..96b04f3b14 --- /dev/null +++ b/families/canary/tests/benchmark/reference.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned NeMo Canary reference for Accuracy qualification.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile +import atexit +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_audio_mono, + resample_linear, + transcription_text, + write_wav_pcm16, +) + + +def _text(value: Any) -> str: + return str(value.text if hasattr(value, "text") else value) + + +def _accuracy(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + from huggingface_hub import snapshot_download + from nemo.collections.asr.models import ASRModel + + model_id, revision = speech_io.model_id(request) + snapshot = snapshot_download( + repo_id=model_id, + revision=revision, + allow_patterns=["*.nemo"], + ) + archives = sorted(Path(snapshot).glob("*.nemo")) + if not archives: + raise FileNotFoundError(f"Canary NeMo archive is missing for {model_id}") + model = ASRModel.restore_from(str(archives[0]), map_location="cpu").eval().to("cuda") + values = model.transcribe([str(sample.wav_path) for sample in samples], batch_size=1) + if isinstance(values, tuple): + values = values[0] + speech_io.write_result(arguments.output, samples, [_text(value) for value in values]) + return 0 + + +def _performance_session( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager" or arguments.precision != "fp32": + raise ValueError("Canary reference requires pytorch-eager fp32") + from huggingface_hub import snapshot_download + from nemo.collections.asr.models import ASRModel + + snapshot = snapshot_download( + repo_id=arguments.model, + revision=arguments.revision, + allow_patterns=["*.nemo"], + local_files_only=arguments.local_files_only, + ) + archives = sorted(Path(snapshot).glob("*.nemo")) + if not archives: + raise FileNotFoundError(f"Canary NeMo archive is missing for {arguments.model}") + model = ASRModel.restore_from(str(archives[0]), map_location="cpu").cpu().eval() + audio, source_rate = load_audio_mono(str(request["audio_path"])) + target_rate = 16_000 + audio = resample_linear(audio, source_rate, target_rate) + temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + temporary.close() + atexit.register(Path(temporary.name).unlink, missing_ok=True) + write_wav_pcm16(Path(temporary.name), audio, target_rate) + + def invoke() -> Mapping[str, Any]: + values = model.transcribe([temporary.name], batch_size=1) + return {"text": transcription_text(values), "output_tokens": None} + + return reference_harness.Session( + invoke, + "nemo", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml b/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml index a2df47f216..bc2f31568a 100644 --- a/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml +++ b/families/chronos_bolt/tests/benchmark/chronos-bolt-tiny-official.yaml @@ -7,6 +7,7 @@ model: chronos-bolt-tiny-official candidate: family: chronos_bolt checkpoint: amazon/chronos-bolt-tiny + revision: a0e552de83495b5c28c14c71c374f3e33280b340 task: time_series_forecast precision: fp32 bundle: chronos-bolt-tiny-official.bundle @@ -57,6 +58,7 @@ performance: reference: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: chronos-bolt} mode: torch-compile fallback: pytorch-eager precision: fp32 diff --git a/families/codegen/tests/benchmark/codegen-350m.yaml b/families/codegen/tests/benchmark/codegen-350m.yaml new file mode 100644 index 0000000000..d7c61e6a73 --- /dev/null +++ b/families/codegen/tests/benchmark/codegen-350m.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: codegen-350m +candidate: + family: codegen + checkpoint: Salesforce/codegen-350M-mono + revision: d9107f71cca463240db1143f4a75a927a27fcb27 + task: text_generation + precision: fp16 + bundle: codegen-350m.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: humaneval-continuation + benchmark: humaneval_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 1.0 + allowed_failures: 0 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: "def fibonacci(n):" + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/convbert/tests/benchmark/convbert-base.yaml b/families/convbert/tests/benchmark/convbert-base.yaml new file mode 100644 index 0000000000..2c34168342 --- /dev/null +++ b/families/convbert/tests/benchmark/convbert-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: convbert-base +candidate: + family: convbert + checkpoint: YituTech/conv-bert-base + revision: 5cb451936b5c4a96562d8b146de85f64f9cf2c22 + task: encoding + precision: fp16 + bundle: convbert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/deberta/tests/benchmark/deberta-base.yaml b/families/deberta/tests/benchmark/deberta-base.yaml new file mode 100644 index 0000000000..87adcb67f4 --- /dev/null +++ b/families/deberta/tests/benchmark/deberta-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deberta-base +candidate: + family: deberta + checkpoint: microsoft/deberta-base + revision: 0d1b43ccf21b5acd9f4e5f7b077fa698f05cf195 + task: encoding + precision: fp16 + bundle: deberta-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml new file mode 100644 index 0000000000..b79012a2ae --- /dev/null +++ b/families/deepseek_ocr/tests/benchmark/deepseek-ocr.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deepseek-ocr +candidate: + family: deepseek_ocr + checkpoint: deepseek-ai/DeepSeek-OCR-2 + revision: aaa02f3811945a91062062994c5c4a3f4c0af2b0 + task: vision_language_generation + precision: fp16 + trust_remote_code: true + bundle: deepseek-ocr.bundle + build: + max_sequence_length: 4096 + fp32_layers: [6, 7, 8, 9, 10, 11, 12] + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: ocrbench-v2-parity + benchmark: ocrbench_v2_parity + samples: 10 + request: + max_new_tokens: 300 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: bf16 + gate: + max_normalized_edit_distance: 0.5 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_pipeline_performance + operation: generate + request: + image_path: ../data/orc_test_img.jpeg + prompt: Extract the text from this image. + max_new_tokens: 300 + temperature: 1.0 + top_k: 1 + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: bf16 + output_contract: ocr-text + max_normalized_edit_distance: 0.5 + required_substrings: + - Architecture + - "Attention:Standard Q/K/V/O" + - "RoPE:Standard rotary position embeddings" + - "Layer 0:Dense SwiGLU MLP" + - "Layers 1-11:MoE" + - "Vision: SAM ViT-B + Qwen2 encoder" diff --git a/families/deepseek_ocr/tests/benchmark/reference.py b/families/deepseek_ocr/tests/benchmark/reference.py new file mode 100644 index 0000000000..eb4076c6a5 --- /dev/null +++ b/families/deepseek_ocr/tests/benchmark/reference.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official DeepSeek OCR reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import atexit +import json +import os +from pathlib import Path +import shutil +import sys +import tempfile +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _model_directory(model: str, revision: str | None) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ), + ) + ) + + +def _run_accuracy(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModel, AutoTokenizer + + if not torch.cuda.is_available(): + raise RuntimeError("DeepSeek OCR Accuracy reference requires CUDA") + if request.get("precision") != "bf16": + raise ValueError("DeepSeek OCR official reference requires bf16") + model_dir = _model_directory(str(request["model"]), request.get("revision")) + tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) + model = ( + AutoModel.from_pretrained( + model_dir, + trust_remote_code=True, + use_safetensors=True, + torch_dtype=torch.bfloat16, + attn_implementation="eager", + ) + .to("cuda") + .eval() + ) + controls = request.get("request", {}) + if not isinstance(controls, dict): + raise ValueError("reference request controls must be an object") + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + results = [] + with tempfile.TemporaryDirectory(prefix="deepseek-ocr-accuracy-") as output_dir: + for sample in samples: + prompt = str(sample["prompt"]) + if "" not in prompt: + prompt = f"\n{prompt}" + text = model.infer( + tokenizer, + prompt=prompt, + image_file=str(sample["image_path"]), + output_path=output_dir, + base_size=1024, + image_size=768, + crop_mode=True, + save_results=False, + eval_mode=True, + ) + results.append( + { + "sample_id": str(sample["sample_id"]), + "text": str(text or ""), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _load_performance( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from transformers import AutoModel, AutoTokenizer + + if arguments.precision != "bf16": + raise ValueError("DeepSeek OCR official reference requires bf16") + model_dir = _model_directory(arguments.model, arguments.revision) + tokenizer = AutoTokenizer.from_pretrained( + model_dir, + trust_remote_code=arguments.trust_remote_code, + local_files_only=arguments.local_files_only, + ) + model = ( + AutoModel.from_pretrained( + model_dir, + trust_remote_code=arguments.trust_remote_code, + local_files_only=arguments.local_files_only, + use_safetensors=True, + torch_dtype=torch.bfloat16, + attn_implementation="eager", + ) + .to("cuda") + .eval() + ) + image_path = str(request["image_path"]) + prompt = str(request.get("prompt", "")) + if "" not in prompt: + prompt = f"\n{prompt}" + output_directory = Path(tempfile.mkdtemp(prefix="deepseek-ocr-performance-")) + atexit.register(shutil.rmtree, output_directory, ignore_errors=True) + + def invoke() -> Mapping[str, Any]: + text = str( + model.infer( + tokenizer, + prompt=prompt, + image_file=image_path, + output_path=str(output_directory), + base_size=1024, + image_size=768, + crop_mode=True, + save_results=False, + eval_mode=True, + ) + or "" + ) + token_ids = tokenizer(text, add_special_tokens=False).input_ids + return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} + + return reference_harness.Session( + invoke, + "transformers", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if "--request" in arguments: + return _run_accuracy(arguments) + return reference_harness.run(arguments, description=__doc__, load=_load_performance) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/deepseek_ocr/tests/test_benchmark_reference.py b/families/deepseek_ocr/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..6823e9e8c8 --- /dev/null +++ b/families/deepseek_ocr/tests/test_benchmark_reference.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/deepseek-ocr.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert reference["output_contract"] == "ocr-text" + assert "adapter" not in reference diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml new file mode 100644 index 0000000000..423fdde74e --- /dev/null +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-lite.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deepseek-v2-lite +candidate: + family: deepseek_v2 + checkpoint: deepseek-ai/DeepSeek-V2-Lite + revision: 604d5664dddd88a0433dbae533b7fe9472482de0 + task: text_generation + precision: fp16 + bundle: deepseek-v2-lite.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 10 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + experts_implementation: batched_mm + output_contract: exact-token-ids diff --git a/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml b/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml new file mode 100644 index 0000000000..efa2c36759 --- /dev/null +++ b/families/deepseek_v2/tests/benchmark/deepseek-v2-tiny.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: deepseek-v2-tiny +candidate: + family: deepseek_v2 + checkpoint: katuni4ka/tiny-random-deepseek-v3 + revision: ba144b0d3331a5892aa588d82722d382be2b6e6b + task: text_generation + precision: fp16 + bundle: deepseek-v2-tiny.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 10 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + experts_implementation: batched_mm + output_contract: exact-token-ids diff --git a/families/detr/tests/benchmark/detr-resnet-50.yaml b/families/detr/tests/benchmark/detr-resnet-50.yaml new file mode 100644 index 0000000000..805afa31cf --- /dev/null +++ b/families/detr/tests/benchmark/detr-resnet-50.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: detr-resnet-50 +candidate: + family: detr + checkpoint: facebook/detr-resnet-50 + revision: 1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b + task: object_detection + precision: fp16 + bundle: detr-resnet-50.bundle + build: + max_sequence_length: 1 + image_height: 1333 + image_width: 1333 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-category-id + request: &detection-request + score_threshold: 0.5 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.5 + max_score_abs_error: 0.05 diff --git a/families/detr/tests/benchmark/reference.py b/families/detr/tests/benchmark/reference.py new file mode 100644 index 0000000000..75e3a71612 --- /dev/null +++ b/families/detr/tests/benchmark/reference.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers DETR reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def _accuracy(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForObjectDetection + + model_id = str(request["model"]) + revision = request.get("revision") + options = {"revision": revision} if revision else {} + processor = AutoImageProcessor.from_pretrained(model_id, **options) + model = ( + AutoModelForObjectDetection.from_pretrained( + model_id, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + **options, + ) + .eval() + .to("cuda") + ) + score_threshold = float(request.get("request", {}).get("score_threshold", 0.5)) + result = [] + for sample in samples: + image = Image.open(str(sample["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + with torch.inference_mode(): + outputs = model(**inputs) + detections = processor.post_process_object_detection( + outputs, + threshold=score_threshold, + target_sizes=torch.tensor([[image.height, image.width]], device=model.device), + )[0] + boxes = detections["boxes"].float().cpu().tolist() + result.append( + { + "sample_id": str(sample["sample_id"]), + "image_height": image.height, + "image_width": image.width, + "boxes": boxes, + "scores": detections["scores"].float().cpu().tolist(), + "class_ids": detections["labels"].cpu().tolist(), + } + ) + arguments.output.write_text(json.dumps({"samples": result}, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _performance_session( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForObjectDetection + + if arguments.mode != "hf-eager": + raise ValueError("DETR reference requires hf-eager mode") + image = Image.open(str(request["image_path"])).convert("RGB") + options = {"revision": arguments.revision} if arguments.revision else {} + processor = AutoImageProcessor.from_pretrained(arguments.model, **options) + model = ( + AutoModelForObjectDetection.from_pretrained( + arguments.model, + torch_dtype=_dtype(torch, arguments.precision), + **options, + ) + .eval() + .to("cuda") + ) + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + threshold = float(request.get("score_threshold", 0.5)) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + detections = processor.post_process_object_detection( + outputs, + threshold=threshold, + target_sizes=torch.tensor([[image.height, image.width]], device=model.device), + )[0] + scores = detections["scores"].float().cpu() + return { + "detected_images": 1, + "detections": int(scores.shape[0]), + "image_height": image.height, + "image_width": image.width, + "boxes": detections["boxes"].float().cpu().reshape(-1).tolist(), + "scores": scores.tolist(), + "class_ids": detections["labels"].cpu().tolist(), + "shape": [int(scores.shape[0]), 4], + "coordinates": "xyxy", + "units": "pixels", + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/detr/tests/cpp/test_image_preprocess_seam.cpp b/families/detr/tests/cpp/test_image_preprocess_seam.cpp index 64a29e7891..23182fb35f 100644 --- a/families/detr/tests/cpp/test_image_preprocess_seam.cpp +++ b/families/detr/tests/cpp/test_image_preprocess_seam.cpp @@ -86,6 +86,19 @@ void test_detr_preprocess_fits_half_tie_engine_dimensions() { check(portrait.size() == 3U * 1333U * 666U, "detr portrait fits reference dimensions"); } +void test_detr_square_engine_accepts_both_image_orientations() { + trtmc::DetrPreprocessConfig config; + config.input_image_h = 1333; + config.input_image_w = 1333; + + const auto landscape = trtmc::compute_detr_resize_shape(382, 640, config); + const auto portrait = trtmc::compute_detr_resize_shape(640, 382, config); + check(landscape.height == 796 && landscape.width == 1333, + "detr square engine accepts landscape resize"); + check(portrait.height == 1333 && portrait.width == 796, + "detr square engine accepts portrait resize"); +} + void test_detr_preprocess_applies_normalization() { const std::vector pixels(3U * 2U * 2U, 0.75F); trtmc::DetrPreprocessConfig config; @@ -160,6 +173,7 @@ int main() { test_detr_resize_rounds_half_ties_to_even(); test_detr_resize_rounds_non_ties_to_nearest(); test_detr_preprocess_fits_half_tie_engine_dimensions(); + test_detr_square_engine_accepts_both_image_orientations(); test_detr_preprocess_applies_normalization(); test_detr_preprocess_rejects_invalid_config(); test_detr_preprocess_rejects_resize_larger_than_engine_input(); diff --git a/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml new file mode 100644 index 0000000000..b60ff4c0ea --- /dev/null +++ b/families/dinov3/tests/benchmark/dinov3-convnext-tiny-pretrain-lvd1689m.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dinov3-convnext-tiny-pretrain-lvd1689m +candidate: + family: dinov3 + checkpoint: facebook/dinov3-convnext-tiny-pretrain-lvd1689m + revision: 10d30274b4d445111e2d5bf75ac93bbd94db274b + task: image_features + precision: fp16 + bundle: dinov3-convnext-tiny-pretrain-lvd1689m.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: beans-knn-parity + benchmark: beans_image_feature_knn + bank_samples_per_class: 24 + query_samples_per_class: 16 + knn_k: 10 + knn_temperature: 0.07 + reference: + command: reference.py + precision: fp32 + batch_size: 16 + gate: + min_pooler_cosine: 0.999 + min_vector_pass_rate: 1.0 + min_knn_top1_agreement: 0.98 + max_knn_accuracy_drop_from_hf: 0.02 + +performance: + - name: extract-features + benchmark: image_features_performance + operation: extract_features + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: image-features-shape diff --git a/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml new file mode 100644 index 0000000000..cc602ac830 --- /dev/null +++ b/families/dinov3/tests/benchmark/dinov3-vits16-pretrain-lvd1689m.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dinov3-vits16-pretrain-lvd1689m +candidate: + family: dinov3 + checkpoint: facebook/dinov3-vits16-pretrain-lvd1689m + revision: 114c1379950215c8b35dfcd4e90a5c251dde0d32 + task: image_features + precision: fp16 + bundle: dinov3-vits16-pretrain-lvd1689m.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: beans-knn-parity + benchmark: beans_image_feature_knn + bank_samples_per_class: 24 + query_samples_per_class: 16 + knn_k: 10 + knn_temperature: 0.07 + reference: + command: reference.py + precision: fp32 + batch_size: 16 + gate: + min_pooler_cosine: 0.999 + min_vector_pass_rate: 1.0 + min_knn_top1_agreement: 0.98 + max_knn_accuracy_drop_from_hf: 0.02 + +performance: + - name: extract-features + benchmark: image_features_performance + operation: extract_features + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: image-features-shape diff --git a/families/dinov3/tests/benchmark/reference.py b/families/dinov3/tests/benchmark/reference.py new file mode 100644 index 0000000000..d414a8b0d9 --- /dev/null +++ b/families/dinov3/tests/benchmark/reference.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers DINOv3 reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def _accuracy(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModel + + model_id = str(request["model"]) + revision = request.get("revision") + options = {"revision": revision} if revision else {} + processor = AutoImageProcessor.from_pretrained(model_id, **options) + model = ( + AutoModel.from_pretrained( + model_id, torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), **options + ) + .eval() + .to("cuda") + ) + batch_size = int(request.get("batch_size", 16)) + result: list[dict[str, Any]] = [] + for start in range(0, len(samples), batch_size): + batch = samples[start : start + batch_size] + images = [Image.open(str(sample["image_path"])).convert("RGB") for sample in batch] + inputs = processor(images=images, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + with torch.inference_mode(): + pooled = model(**inputs).pooler_output.float().cpu() + for sample, vector in zip(batch, pooled, strict=True): + result.append( + { + "sample_id": str(sample["sample_id"]), + "pooler_output": vector.tolist(), + } + ) + arguments.output.write_text(json.dumps({"samples": result}, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _performance_session( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModel + + if arguments.mode != "hf-eager": + raise ValueError("DINOv3 reference requires hf-eager mode") + options = {"revision": arguments.revision} if arguments.revision else {} + processor = AutoImageProcessor.from_pretrained(arguments.model, **options) + model = ( + AutoModel.from_pretrained( + arguments.model, + torch_dtype=_dtype(torch, arguments.precision), + **options, + ) + .eval() + .to("cuda") + ) + image = Image.open(str(request["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + return { + "last_hidden_state_shape": [int(size) for size in outputs.last_hidden_state.shape], + "pooler_output_shape": [int(size) for size in outputs.pooler_output.shape], + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml b/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml new file mode 100644 index 0000000000..4d10a25223 --- /dev/null +++ b/families/distilbert/tests/benchmark/distilbert-base-uncased.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: distilbert-base-uncased +candidate: + family: distilbert + checkpoint: distilbert/distilbert-base-uncased + revision: 12040accade4e8a0f71eabdb258fecc2e7e948be + task: encoding + precision: fp16 + bundle: distilbert-base-uncased.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml new file mode 100644 index 0000000000..cf2d0c9807 --- /dev/null +++ b/families/dpr/tests/benchmark/dpr-ctx-encoder.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dpr-ctx-encoder +candidate: + family: dpr + checkpoint: facebook/dpr-ctx_encoder-single-nq-base + revision: bb21a3c2b1656d60c6a8e920283bc40dabddadb8 + task: encoding + precision: fp16 + bundle: dpr-ctx-encoder.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + model_class: transformers.DPRContextEncoder + tokenizer_class: transformers.DPRContextEncoderTokenizerFast + gate: + min_vector_cosine: 0.99 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml b/families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml new file mode 100644 index 0000000000..ed61663485 --- /dev/null +++ b/families/eagle_vlm/tests/benchmark/nemotron-embed-vl-1b-v2.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-embed-vl-1b-v2 +candidate: + family: eagle_vlm + checkpoint: nvidia/llama-nemotron-embed-vl-1b-v2 + revision: 582e3bf72aee355e3c59ed89de53543c5b0657ee + task: embedding + precision: fp16 + trust_remote_code: true + bundle: nemotron-embed-vl-1b-v2.bundle + build: + max_sequence_length: 512 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.995 + min_vector_pass_rate: 0.98 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: embed + benchmark: embedding_performance + operation: embed + request: + prompt: What is artificial intelligence? + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-embedding + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + reference_backend: hf_transformers + precision: fp32 + output_contract: embedding-shape diff --git a/families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml b/families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml new file mode 100644 index 0000000000..f2759dc3ad --- /dev/null +++ b/families/eagle_vlm/tests/benchmark/nemotron-rerank-vl-1b-v2.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-rerank-vl-1b-v2 +candidate: + family: eagle_vlm + checkpoint: nvidia/llama-nemotron-rerank-vl-1b-v2 + revision: 9e95da054312436dfc703319dd2b793a3bee2465 + task: reranking + precision: fp16 + trust_remote_code: true + bundle: nemotron-rerank-vl-1b-v2.bundle + build: + max_sequence_length: 685 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: stsbenchmark-reranking-parity + benchmark: stsbenchmark_reranking_parity + samples: 10 + reference: + command: reranking_reference.py + precision: fp32 + gate: + max_score_abs_error: 0.05 + min_sample_pass_rate: 0.9 + +performance: + - name: rerank + benchmark: reranking_performance + operation: rerank + request: + query: 0-dimensional biomaterials show inductive properties. + documents: + - Nanotechnologies provide new ways to manipulate and track stem cells. + - Depression is associated with immune dysregulation. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-reranking + mode: hf-eager + reference_backend: hf_transformers + precision: fp16 + output_contract: reranking-order diff --git a/families/eagle_vlm/tests/benchmark/reranking_reference.py b/families/eagle_vlm/tests/benchmark/reranking_reference.py new file mode 100644 index 0000000000..28f292945d --- /dev/null +++ b/families/eagle_vlm/tests/benchmark/reranking_reference.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Nemotron VL reranking reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + + +def _model_directory(model: str, revision: str | None) -> Path: + from huggingface_hub import snapshot_download + + offline = ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=offline, + ) + ) + + +def _dtype(torch_module: Any, precision: str) -> Any: + try: + return { + "bf16": torch_module.bfloat16, + "fp16": torch_module.float16, + "fp32": torch_module.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModelForSequenceClassification, AutoProcessor + + model_dir = _model_directory(str(request["model"]), request.get("revision")) + processor = AutoProcessor.from_pretrained( + model_dir, + trust_remote_code=True, + max_input_tiles=6, + use_thumbnail=True, + rerank_max_length=8192, + ) + model = ( + AutoModelForSequenceClassification.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + ) + .eval() + .to("cuda") + ) + results = [] + with torch.inference_mode(): + for sample in request["samples"]: + documents = [str(value) for value in sample["documents"]] + examples = [ + { + "question": str(sample["query"]), + "doc_text": document, + "doc_image": "", + } + for document in documents + ] + encoded = processor.process_queries_documents_crossencoder(examples) + encoded = { + name: value.to("cuda") if hasattr(value, "to") else value + for name, value in encoded.items() + } + logits = model(**encoded).logits.float().cpu() + if logits.ndim == 2: + logits = logits[:, 0] if logits.shape[-1] == 1 else logits[:, -1] + scores = [float(value) for value in logits.reshape(-1).tolist()] + if len(scores) != len(documents): + raise RuntimeError("reranking reference returned the wrong score count") + results.append({"sample_id": str(sample["sample_id"]), "scores": scores}) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/electra/tests/benchmark/electra-base-discriminator.yaml b/families/electra/tests/benchmark/electra-base-discriminator.yaml new file mode 100644 index 0000000000..22b4ecf591 --- /dev/null +++ b/families/electra/tests/benchmark/electra-base-discriminator.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: electra-base-discriminator +candidate: + family: electra + checkpoint: google/electra-base-discriminator + revision: 1ae76a97c7e84a4e640876a07453fccd636f0667 + task: encoding + precision: fp16 + bundle: electra-base-discriminator.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/fast_foundation_stereo/support.py b/families/fast_foundation_stereo/support.py index 83cb7ffdbe..6d255987d5 100644 --- a/families/fast_foundation_stereo/support.py +++ b/families/fast_foundation_stereo/support.py @@ -8,6 +8,7 @@ describe = family_support( model_types=("fast-foundation-stereo", "fast_foundation_stereo", "foundation_stereo_lite"), + required_files=("weights/23-36-37/model_best_bp2_serialize.pth",), tasks=("stereo_disparity",), default_task="stereo_disparity", ) diff --git a/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml b/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml new file mode 100644 index 0000000000..bb45d77647 --- /dev/null +++ b/families/fast_foundation_stereo/tests/benchmark/fast-foundation-stereo.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: fast-foundation-stereo +candidate: + family: fast_foundation_stereo + checkpoint: nvidia/c-fast-foundationstereo + revision: 9b446878c81ddb27593036767b29b2859d46103e + model_directory: trtmc-reference/Fast-FoundationStereo + task: stereo_disparity + precision: fp16 + bundle: fast-foundation-stereo.bundle + build: + tensor_parallel_size: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: office-disparity-parity + benchmark: stereo_disparity_parity + samples: 1 + request: &disparity-request + height: 700 + width: 700 + max_disp: 192 + valid_iters: 8 + reference: + command: reference.py + precision: fp16 + gate: + min_cosine: 0.999 + max_mean_abs_error: 0.5 + max_bad_2px_fraction: 0.03 + min_sample_pass_rate: 1.0 + +performance: + - name: disparity + benchmark: stereo_disparity_performance + operation: disparity + request: + <<: *disparity-request + left_image_path: ../data/office_left.png + right_image_path: ../data/office_right.png + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: official_pytorch + precision: fp16 + output_contract: disparity-parity + min_disparity_cosine: 0.999 + max_disparity_mean_abs_error: 0.5 + max_disparity_bad_2px_fraction: 0.03 diff --git a/families/fast_foundation_stereo/tests/benchmark/prepare_environment.py b/families/fast_foundation_stereo/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..0a875ee350 --- /dev/null +++ b/families/fast_foundation_stereo/tests/benchmark/prepare_environment.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned official reference implementation inside this family environment.""" + +from __future__ import annotations + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +from huggingface_hub import hf_hub_download + + +SOURCE_REPOSITORY = "https://github.com/NVlabs/Fast-FoundationStereo.git" +SOURCE_REVISION = "a290ba04c1b3ad1ec41a33974a157b2917b624d4" +CHECKPOINT = "nvidia/c-fast-foundationstereo" +CHECKPOINT_REVISION = "9b446878c81ddb27593036767b29b2859d46103e" + + +def main() -> int: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "Fast-FoundationStereo" + required = ( + destination / "core/foundation_stereo.py", + destination / "core/submodule.py", + destination / "weights/23-36-37/model_best_bp2_serialize.pth", + ) + if all(path.is_file() for path in required): + return 0 + if destination.exists(): + raise RuntimeError(f"incomplete Fast Foundation Stereo reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="Fast-FoundationStereo.", dir=parent)) + try: + subprocess.run( + ["git", "init", "--quiet", str(temporary)], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", SOURCE_REPOSITORY], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(temporary), + "fetch", + "--quiet", + "--depth=1", + "origin", + SOURCE_REVISION, + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + checkpoint = Path( + hf_hub_download( + repo_id=CHECKPOINT, + revision=CHECKPOINT_REVISION, + filename="model_best_bp2_serialize.pth", + ) + ) + target = temporary / "weights/23-36-37/model_best_bp2_serialize.pth" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(checkpoint, target) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/fast_foundation_stereo/tests/benchmark/reference.py b/families/fast_foundation_stereo/tests/benchmark/reference.py new file mode 100644 index 0000000000..5077df139b --- /dev/null +++ b/families/fast_foundation_stereo/tests/benchmark/reference.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned official Accuracy and Performance reference for Fast Foundation Stereo.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import statistics +import sys +import time +from typing import Any, Mapping + + +def _model_root() -> Path: + root = Path(sys.prefix) / "trtmc-reference/Fast-FoundationStereo" + required = ( + root / "core/foundation_stereo.py", + root / "core/submodule.py", + root / "weights/23-36-37/model_best_bp2_serialize.pth", + ) + if not all(path.is_file() for path in required): + raise RuntimeError(f"Fast Foundation Stereo reference is incomplete: {root}") + return root + + +def _load_model(max_disparity: int, valid_iters: int): + import torch + + root = _model_root() + previous = Path.cwd() + try: + os.chdir(root) + sys.path.insert(0, str(root)) + from core.utils.utils import InputPadder + from Utils import AMP_DTYPE + + model = torch.load( + root / "weights/23-36-37/model_best_bp2_serialize.pth", + map_location="cpu", + weights_only=False, + ) + finally: + os.chdir(previous) + model.args.max_disp = max_disparity + model.args.valid_iters = valid_iters + model.args.normalize = True + return model.cuda().eval(), InputPadder, AMP_DTYPE + + +def _image(path: str, *, height: int, width: int): + import numpy as np + from PIL import Image + + with Image.open(path) as image: + pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) + if pixels.shape != (height, width, 3): + raise ValueError(f"stereo image must be {height}x{width} RGB, got {pixels.shape}") + return pixels + + +def _inference( + model: Any, + input_padder: Any, + amp_dtype: Any, + left: Any, + right: Any, + *, + height: int, + width: int, + valid_iters: int, +): + import numpy as np + import torch + + left_tensor = torch.as_tensor(left).cuda().float()[None].permute(0, 3, 1, 2) + right_tensor = torch.as_tensor(right).cuda().float()[None].permute(0, 3, 1, 2) + padder = input_padder(left_tensor.shape, divis_by=32, force_square=False) + left_tensor, right_tensor = padder.pad(left_tensor, right_tensor) + with torch.inference_mode(), torch.amp.autocast("cuda", enabled=True, dtype=amp_dtype): + disparity = model.forward( + left_tensor, + right_tensor, + iters=valid_iters, + test_mode=True, + optimize_build_volume="pytorch1", + ) + values = padder.unpad(disparity.float()).cpu().numpy().reshape(height, width) + return np.clip(values, 0, None).astype(" tuple[int, int, int, int]: + height = int(request.get("height", 700)) + width = int(request.get("width", 700)) + max_disparity = int(request.get("max_disp", 192)) + valid_iters = int(request.get("valid_iters", 8)) + if (height, width, max_disparity, valid_iters) != (700, 700, 192, 8): + raise ValueError( + "Fast Foundation Stereo reference requires 700x700, max_disp=192, valid_iters=8" + ) + return height, width, max_disparity, valid_iters + + +def _summary(values: Any, artifact: Path) -> dict[str, Any]: + import numpy as np + + values.tofile(artifact) + return { + "height": int(values.shape[0]), + "width": int(values.shape[1]), + "element_count": int(values.size), + "finite_fraction": float(np.isfinite(values).mean()), + "nonnegative_fraction": float((values >= 0).mean()), + "disparity_artifact": str(artifact.resolve()), + } + + +def _run_accuracy(arguments: argparse.Namespace) -> int: + payload = json.loads(arguments.request.read_text(encoding="utf-8")) + request = payload.get("request", {}) + samples = payload.get("samples") + if not isinstance(request, Mapping) or not isinstance(samples, list) or not samples: + raise ValueError("Accuracy reference requires request and samples") + height, width, max_disparity, valid_iters = _options(request) + model, input_padder, amp_dtype = _load_model(max_disparity, valid_iters) + output_samples = [] + for index, sample in enumerate(samples): + if not isinstance(sample, Mapping): + raise ValueError(f"Accuracy sample {index} must be an object") + left = _image(str(sample["left_image_path"]), height=height, width=width) + right = _image(str(sample["right_image_path"]), height=height, width=width) + values = _inference( + model, + input_padder, + amp_dtype, + left, + right, + height=height, + width=width, + valid_iters=valid_iters, + ) + artifact = arguments.output.with_name( + f"{arguments.output.stem}.{index}.disparity.f32" + ) + output_samples.append( + {"sample_id": str(sample["sample_id"]), **_summary(values, artifact)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": output_samples}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +def _run_performance(arguments: argparse.Namespace) -> int: + import torch + + request = json.loads(arguments.request_json) + timing = json.loads(arguments.timing_contract_json) + height, width, max_disparity, valid_iters = _options(request) + model, input_padder, amp_dtype = _load_model(max_disparity, valid_iters) + left = _image(str(request["left_image_path"]), height=height, width=width) + right = _image(str(request["right_image_path"]), height=height, width=width) + + def invoke(): + return _inference( + model, + input_padder, + amp_dtype, + left, + right, + height=height, + width=width, + valid_iters=valid_iters, + ) + + result = None + for _ in range(arguments.warmup): + result = invoke() + samples_ms = [] + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter_ns() + result = invoke() + torch.cuda.synchronize() + samples_ms.append((time.perf_counter_ns() - started) / 1_000_000) + if result is None: + raise RuntimeError("reference produced no disparity output") + artifact = arguments.output.with_suffix(".disparity.f32") + arguments.output.parent.mkdir(parents=True, exist_ok=True) + summary = _summary(result, artifact) + value = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + "framework": f"torch-{torch.__version__}", + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": timing, + **timing, + "samples_ms": samples_ms, + "metrics": {"latency_ms": {"p50": statistics.median(samples_ms)}}, + "output_summary": summary, + } + arguments.output.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + for name in ( + "model", + "family", + "manifest", + "operation", + "selected-task", + "request-json", + "adapter-options-json", + "timing-contract-json", + "precision", + "mode", + "padding", + "case-name", + ): + parser.add_argument("--" + name, required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + return parser + + +def main() -> int: + if "--request" in sys.argv[1:]: + return _run_accuracy(_accuracy_parser().parse_args()) + return _run_performance(_performance_parser().parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/fast_foundation_stereo/tests/data/benchmark_stereo.json b/families/fast_foundation_stereo/tests/data/benchmark_stereo.json new file mode 100644 index 0000000000..622c4addf6 --- /dev/null +++ b/families/fast_foundation_stereo/tests/data/benchmark_stereo.json @@ -0,0 +1,9 @@ +{ + "requests": [ + { + "id": "office", + "left_image": "office_left.png", + "right_image": "office_right.png" + } + ] +} diff --git a/families/fast_foundation_stereo/tests/test_support.py b/families/fast_foundation_stereo/tests/test_support.py new file mode 100644 index 0000000000..04f0cf9ac5 --- /dev/null +++ b/families/fast_foundation_stereo/tests/test_support.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from tensorrt_model_connect.model_support import ModelMetadata + +from families.fast_foundation_stereo.support import describe + + +def test_prepared_official_model_without_hugging_face_config_is_supported() -> None: + support = describe( + ModelMetadata({}, {}, ("weights/23-36-37/model_best_bp2_serialize.pth",)) + ) + + assert support is not None + assert support.default_task == "stereo_disparity" + + +def test_unrelated_rootless_checkpoint_is_not_supported() -> None: + assert describe(ModelMetadata({}, {}, ("model.pt",))) is None diff --git a/families/flux/tests/benchmark/flux-2-dev-fp8.yaml b/families/flux/tests/benchmark/flux-2-dev-fp8.yaml new file mode 100644 index 0000000000..ecddccfed2 --- /dev/null +++ b/families/flux/tests/benchmark/flux-2-dev-fp8.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: flux-2-dev-fp8 +candidate: + family: flux + checkpoint: black-forest-labs/FLUX.2-dev + revision: 26afe3a78bb242c0a8bb181dcc8937bb16e5c66c + task: image_generation + precision: fp16 + bundle: flux-2-dev-fp8.bundle + build: {quantization: fp8, image_height: 1024, image_width: 1024} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 28, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - name: generate-image + benchmark: media_generation_performance + operation: generate_image + request: *request + measurement: {warmup: 5, iterations: 10} + reference: + <<: *reference + runner: task-reference + reference_backend: hf_diffusers + output_contract: media-shape diff --git a/families/flux/tests/benchmark/flux-2-dev.yaml b/families/flux/tests/benchmark/flux-2-dev.yaml new file mode 100644 index 0000000000..15258d13da --- /dev/null +++ b/families/flux/tests/benchmark/flux-2-dev.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: flux-2-dev +candidate: + family: flux + checkpoint: black-forest-labs/FLUX.2-dev + revision: 26afe3a78bb242c0a8bb181dcc8937bb16e5c66c + task: image_generation + precision: bf16 + bundle: flux-2-dev.bundle + build: {image_height: 1024, image_width: 1024} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 28, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - name: generate-image + benchmark: media_generation_performance + operation: generate_image + request: *request + measurement: {warmup: 5, iterations: 10} + reference: + <<: *reference + runner: task-reference + reference_backend: hf_diffusers + output_contract: media-shape diff --git a/families/flux/tests/benchmark/flux-schnell.yaml b/families/flux/tests/benchmark/flux-schnell.yaml new file mode 100644 index 0000000000..c15b8e36f4 --- /dev/null +++ b/families/flux/tests/benchmark/flux-schnell.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: flux-schnell +candidate: + family: flux + checkpoint: black-forest-labs/FLUX.1-schnell + revision: 741f7c3ce8b383c54771c7003378a50191e9efe9 + task: image_generation + precision: fp16 + bundle: flux-schnell.bundle + build: {image_height: 1024, image_width: 1024} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - &case + name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 4, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: fp16, output_contract: media-artifact-parity} + gate: &gate {min_psnr: 5.0, min_ssim: 0.1} +performance: + - name: generate-image + benchmark: media_generation_performance + operation: generate_image + request: *request + measurement: {warmup: 5, iterations: 10} + reference: + <<: *reference + runner: task-reference + reference_backend: hf_diffusers + output_contract: media-shape diff --git a/families/fnet/tests/benchmark/fnet-base.yaml b/families/fnet/tests/benchmark/fnet-base.yaml new file mode 100644 index 0000000000..4a32f80d6b --- /dev/null +++ b/families/fnet/tests/benchmark/fnet-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: fnet-base +candidate: + family: fnet + checkpoint: google/fnet-base + revision: d89b6fad3cf5384848b783dc480f9685f49d008c + task: encoding + precision: fp16 + bundle: fnet-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/gemma/tests/benchmark/gemma-2-2b.yaml b/families/gemma/tests/benchmark/gemma-2-2b.yaml new file mode 100644 index 0000000000..b70117cd12 --- /dev/null +++ b/families/gemma/tests/benchmark/gemma-2-2b.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gemma-2-2b +candidate: + family: gemma + checkpoint: google/gemma-2-2b-it + revision: 299a8560bedf22ed1c72a8a11e7dce4a7f9f51f8 + task: text_generation + precision: fp16 + bundle: gemma-2-2b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is 2 + 2? Answer with just the number. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/gemma/tests/benchmark/gemma-3-1b.yaml b/families/gemma/tests/benchmark/gemma-3-1b.yaml new file mode 100644 index 0000000000..0c2e2dc843 --- /dev/null +++ b/families/gemma/tests/benchmark/gemma-3-1b.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gemma-3-1b +candidate: + family: gemma + checkpoint: google/gemma-3-1b-it + revision: dcc83ea841ab6100d6b47a070329e1ba4cf78752 + task: text_generation + precision: bf16 + bundle: gemma-3-1b.bundle + build: {max_sequence_length: 257} +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: {precision: fp32} + request: &greedy + {max_new_tokens: 64, temperature: 0.0, top_k: 1, top_p: 1.0, min_p: 0.0, seed: -1, repetition_penalty: 1.0, use_chat_template: false, enable_thinking: false, do_sample: false} + gate: {min_pass_rate: 0.9, allowed_failures: 1} +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy + prompt: What is the capital of France? Answer with just the name. + max_new_tokens: 10 + use_chat_template: true + measurement: {warmup: 5, iterations: 10} + reference: {runner: hf-transformers, mode: torch-compile, fallback: hf-eager, compile_scope: model.forward, precision: fp32, output_contract: exact-token-ids} diff --git a/families/gemma/tests/benchmark/gemma-3-270m.yaml b/families/gemma/tests/benchmark/gemma-3-270m.yaml new file mode 100644 index 0000000000..eaca7fbb04 --- /dev/null +++ b/families/gemma/tests/benchmark/gemma-3-270m.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gemma-3-270m +candidate: &candidate + family: gemma + checkpoint: google/gemma-3-270m-it + revision: ac82b4e820549b854eebf28ce6dedaf9fdfa17b3 + task: text_generation + precision: bf16 + bundle: gemma-3-270m.bundle + build: + max_sequence_length: 257 +accuracy: &accuracy + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: {precision: fp32} + request: &greedy + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: {min_pass_rate: 0.9, allowed_failures: 1} +performance: &performance + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy + prompt: What is the capital of France? Answer with just the name. + max_new_tokens: 10 + use_chat_template: true + measurement: {warmup: 5, iterations: 10} + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/gemma/tests/benchmark/gemma-3-4b.yaml b/families/gemma/tests/benchmark/gemma-3-4b.yaml new file mode 100644 index 0000000000..0b6d94cc70 --- /dev/null +++ b/families/gemma/tests/benchmark/gemma-3-4b.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gemma-3-4b +candidate: + family: gemma + checkpoint: google/gemma-3-4b-it + revision: 093f9f388b31de276ce2de164bdc2081324b9767 + task: text_generation + precision: bf16 + bundle: gemma-3-4b.bundle + build: {max_sequence_length: 257} +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: {precision: fp32} + request: &greedy + {max_new_tokens: 64, temperature: 0.0, top_k: 1, top_p: 1.0, min_p: 0.0, seed: -1, repetition_penalty: 1.0, use_chat_template: false, enable_thinking: false, do_sample: false} + gate: {min_pass_rate: 0.9, allowed_failures: 1} +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy + prompt: What is the capital of France? Answer with just the name. + max_new_tokens: 10 + use_chat_template: true + measurement: {warmup: 5, iterations: 10} + reference: {runner: hf-transformers, mode: torch-compile, fallback: hf-eager, compile_scope: model.forward, precision: fp32, output_contract: exact-token-ids} diff --git a/families/falcon/tests/benchmark/falcon-rw-1b.yaml b/families/glm/tests/benchmark/glm-4-9b.yaml similarity index 88% rename from families/falcon/tests/benchmark/falcon-rw-1b.yaml rename to families/glm/tests/benchmark/glm-4-9b.yaml index a16e55b83a..2593f65670 100644 --- a/families/falcon/tests/benchmark/falcon-rw-1b.yaml +++ b/families/glm/tests/benchmark/glm-4-9b.yaml @@ -2,13 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: falcon-rw-1b +model: glm-4-9b candidate: - family: falcon - checkpoint: tiiuae/falcon-rw-1b + family: glm + checkpoint: THUDM/glm-4-9b-hf + revision: b44e98fcc8df0faba03a48b405356af6b91821e7 task: text_generation - precision: fp32 - bundle: falcon-rw-1b.bundle + precision: fp16 + bundle: glm-4-9b.bundle build: max_sequence_length: 256 diff --git a/families/glmasr/requirements.txt b/families/glmasr/requirements.txt new file mode 100644 index 0000000000..be9ba894f6 --- /dev/null +++ b/families/glmasr/requirements.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +scipy==1.15.3 +soundfile==0.14.0 +transformers==5.2.0 diff --git a/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml b/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml new file mode 100644 index 0000000000..168766e944 --- /dev/null +++ b/families/glmasr/tests/benchmark/glmasr-nano-fp16.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: glmasr-nano-fp16 +candidate: + family: glmasr + checkpoint: zai-org/GLM-ASR-Nano-2512 + revision: 61ba4e0b3309b6656edea3e93e419f7bd5c61957 + task: transcription + precision: fp16 + bundle: glmasr-nano-fp16.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 72 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/librispeech-test-clean-6930-75918-0003.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/glmasr/tests/benchmark/reference.py b/families/glmasr/tests/benchmark/reference.py new file mode 100644 index 0000000000..2d960224bb --- /dev/null +++ b/families/glmasr/tests/benchmark/reference.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official GLM-ASR reference for Accuracy qualification.""" + +from __future__ import annotations + +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_audio_mono, + resample_linear, +) + + +def _accuracy(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + import torch + from transformers import GlmAsrForConditionalGeneration, GlmAsrProcessor + + model_id, revision = speech_io.model_id(request) + options = {"revision": revision} if revision else {} + processor = GlmAsrProcessor.from_pretrained(model_id, **options) + model = ( + GlmAsrForConditionalGeneration.from_pretrained( + model_id, + torch_dtype=speech_io.torch_dtype(torch, request), + **options, + ) + .eval() + .to("cuda") + ) + max_new_tokens = int(request.get("max_new_tokens", 128)) + texts = [] + for sample in samples: + conversation = [ + { + "role": "user", + "content": [ + {"type": "audio", "audio": sample.waveform}, + {"type": "text", "text": "Please transcribe this audio into text"}, + ], + } + ] + inputs = processor.apply_chat_template( + conversation, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, + sampling_rate=sample.sample_rate, + ).to(model.device) + with torch.inference_mode(): + generated = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) + new_tokens = generated[0, inputs["input_ids"].shape[1] :] + texts.append(processor.tokenizer.decode(new_tokens, skip_special_tokens=True)) + speech_io.write_result(arguments.output, samples, texts) + return 0 + + +def _performance_session( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "hf-eager": + raise ValueError("GLM-ASR reference requires hf-eager mode") + import torch + from transformers import GlmAsrForConditionalGeneration, GlmAsrProcessor + + options = {"revision": arguments.revision} if arguments.revision else {} + processor = GlmAsrProcessor.from_pretrained(arguments.model, **options) + dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[ + arguments.precision + ] + model = ( + GlmAsrForConditionalGeneration.from_pretrained( + arguments.model, torch_dtype=dtype, **options + ) + .eval() + .to("cuda") + ) + audio, source_rate = load_audio_mono(str(request["audio_path"])) + target_rate = 16_000 + audio = resample_linear(audio, source_rate, target_rate) + conversation = [ + { + "role": "user", + "content": [ + {"type": "audio", "audio": audio}, + {"type": "text", "text": "Please transcribe this audio into text"}, + ], + } + ] + inputs = processor.apply_chat_template( + conversation, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, + sampling_rate=target_rate, + ).to(model.device) + prompt_tokens = int(inputs["input_ids"].shape[1]) + max_new_tokens = int(request.get("max_new_tokens", 128)) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + generated = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + ) + token_ids = [int(token) for token in generated[0, prompt_tokens:].cpu().tolist()] + return { + "text": processor.tokenizer.decode(token_ids, skip_special_tokens=True).strip(), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/mixtral/tests/benchmark/mixtral-stories-15m.yaml b/families/gpt2/tests/benchmark/distilgpt2.yaml similarity index 75% rename from families/mixtral/tests/benchmark/mixtral-stories-15m.yaml rename to families/gpt2/tests/benchmark/distilgpt2.yaml index 83b4cb1376..221c6e469c 100644 --- a/families/mixtral/tests/benchmark/mixtral-stories-15m.yaml +++ b/families/gpt2/tests/benchmark/distilgpt2.yaml @@ -2,20 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: mixtral-stories-15m +model: distilgpt2 candidate: - family: mixtral - checkpoint: ggml-org/stories15M_MOE + family: gpt2 + checkpoint: distilbert/distilgpt2 + revision: 2290a62682d06624634c1f46a6ad5be0f47f38aa task: text_generation precision: fp16 - bundle: mixtral-stories-15m.bundle + bundle: distilgpt2.bundle build: - max_sequence_length: 256 - fp32_layers: [3, 4, 5] + max_sequence_length: 257 accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation + - name: wikitext103-continuation + benchmark: wikitext103_continuation samples: 10 prompt_token_limit: 192 truncation_side: left @@ -37,12 +37,13 @@ accuracy: allowed_failures: 1 performance: - - name: generate-64 + - name: generate-12 benchmark: text_generation_performance operation: generate request: <<: *greedy-generation - prompt: Once upon a time there was a little + prompt: Hello, I'm a language model + max_new_tokens: 12 measurement: warmup: 5 iterations: 10 diff --git a/families/gpt2/tests/benchmark/gpt2-125m.yaml b/families/gpt2/tests/benchmark/gpt2-125m.yaml index 447178191c..cc920726ff 100644 --- a/families/gpt2/tests/benchmark/gpt2-125m.yaml +++ b/families/gpt2/tests/benchmark/gpt2-125m.yaml @@ -7,6 +7,7 @@ model: gpt2-125m candidate: family: gpt2 checkpoint: openai-community/gpt2 + revision: 607a30d783dfa663caf39e06633721c8d4cfcd7e task: text_generation precision: fp32 bundle: gpt2-125m.bundle diff --git a/families/gpt_neox/tests/benchmark/pythia-70m.yaml b/families/gpt_neox/tests/benchmark/pythia-70m.yaml deleted file mode 100644 index a80f8dda2e..0000000000 --- a/families/gpt_neox/tests/benchmark/pythia-70m.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: pythia-70m -candidate: - family: gpt_neox - checkpoint: EleutherAI/pythia-70m - task: text_generation - precision: fp32 - bundle: pythia-70m.bundle - build: - max_sequence_length: 400 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 336 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-64 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: The capital of France is - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml b/families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml new file mode 100644 index 0000000000..395991d511 --- /dev/null +++ b/families/gpt_oss/tests/benchmark/gpt-oss-20b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: gpt-oss-20b +candidate: + family: gpt_oss + checkpoint: openai/gpt-oss-20b + revision: 6cee5e81ee83917806bbde320786a8fb61efebee + task: text_generation + precision: fp16 + bundle: gpt-oss-20b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: <|start|>system<|message|>You are a helpful assistant.<|end|><|start|>user<|message|>What is the capital of France?<|end|><|start|>assistant + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/internlm/tests/benchmark/internlm2-1.8b.yaml b/families/internlm/tests/benchmark/internlm2-1.8b.yaml new file mode 100644 index 0000000000..97845c085b --- /dev/null +++ b/families/internlm/tests/benchmark/internlm2-1.8b.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: internlm2-1.8b +candidate: + family: internlm + checkpoint: internlm/internlm2-math-plus-1_8b + revision: 998dd540b92711b855982c0318a39064a1d3ddc1 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: internlm2-1.8b.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 20 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/internvl/tests/benchmark/internvl3-2b.yaml b/families/internvl/tests/benchmark/internvl3-2b.yaml new file mode 100644 index 0000000000..256c1656f8 --- /dev/null +++ b/families/internvl/tests/benchmark/internvl3-2b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: internvl3-2b +candidate: + family: internvl + checkpoint: OpenGVLab/InternVL3-2B-hf + revision: cb57a075cb75a2e6d1b668b128d48bb00ae321d2 + task: vision_language_generation + precision: fp16 + bundle: internvl3-2b-vl.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/internvl/tests/benchmark/internvl3-8b.yaml b/families/internvl/tests/benchmark/internvl3-8b.yaml new file mode 100644 index 0000000000..bb7bcaf2e3 --- /dev/null +++ b/families/internvl/tests/benchmark/internvl3-8b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: internvl3-8b +candidate: + family: internvl + checkpoint: OpenGVLab/InternVL3-8B-hf + revision: 259a3b64a14623c0ec91a045cb43f7c5af5fa6af + task: vision_language_generation + precision: fp16 + bundle: internvl3-8b-vl.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/internvl/tests/benchmark/reference.py b/families/internvl/tests/benchmark/reference.py new file mode 100644 index 0000000000..7194f1a412 --- /dev/null +++ b/families/internvl/tests/benchmark/reference.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers InternVL reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import AutoModelForImageTextToText, AutoProcessor + + try: + dtype = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True) + model = ( + AutoModelForImageTextToText.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=dtype, + ) + .eval() + .to("cuda") + ) + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + rendered = reference.processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": prompt}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + if ( + not isinstance(rendered, str) + or not rendered.strip() + or prompt not in rendered + or "" not in rendered + ): + raise ValueError("InternVL chat template lost the image or text prompt") + encoded = reference.processor(text=rendered, images=image, padding=True, return_tensors="pt") + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if hasattr(value, "to") + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any], max_new_tokens: int) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + generated = reference.model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = generated[0] + tokens = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in tokens.detach().cpu().tolist()] + return { + "text": reference.processor.decode(tokens, skip_special_tokens=True), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + prompt = str(controls.get("prompt", "")) + max_new_tokens = int(controls.get("max_new_tokens", 10)) + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append( + { + "sample_id": str(sample["sample_id"]), + **_infer(reference, inputs, max_new_tokens), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "internvl" or arguments.operation != "generate": + raise ValueError("InternVL reference only supports internvl generate") + if arguments.mode != "pytorch-eager": + raise ValueError("InternVL reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request.get("prompt", ""))) + max_new_tokens = int(request.get("max_new_tokens", 10)) + for _ in range(arguments.warmup): + _infer(reference, inputs, max_new_tokens) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs, max_new_tokens) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/mamba/tests/benchmark/mamba-130m.yaml b/families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml similarity index 73% rename from families/mamba/tests/benchmark/mamba-130m.yaml rename to families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml index ebe75ab705..a26cd9a284 100644 --- a/families/mamba/tests/benchmark/mamba-130m.yaml +++ b/families/k2_horizon/tests/benchmark/k2-horizon-7b.yaml @@ -2,15 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: mamba-130m +model: k2-horizon-7b candidate: - family: mamba - checkpoint: state-spaces/mamba-130m-hf + family: k2_horizon + checkpoint: IFM/K2-Horizon-7B + revision: 586b03f0fd1fbbf2f13eeafc33749e95ae34dd10 task: text_generation - precision: fp32 - bundle: mamba-130m.bundle + precision: bf16 + trust_remote_code: true + bundle: k2-horizon-7b.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt accuracy: - name: mmlu-continuation @@ -19,7 +24,7 @@ accuracy: prompt_token_limit: 192 truncation_side: left reference: - precision: fp32 + precision: bf16 request: &greedy-generation max_new_tokens: 64 temperature: 0.0 @@ -36,12 +41,13 @@ accuracy: allowed_failures: 1 performance: - - name: generate-64 + - name: generate-4 benchmark: text_generation_performance operation: generate request: <<: *greedy-generation prompt: The capital of France is + max_new_tokens: 4 measurement: warmup: 5 iterations: 10 @@ -50,5 +56,5 @@ performance: mode: torch-compile fallback: hf-eager compile_scope: model.forward - precision: fp32 + precision: bf16 output_contract: exact-token-ids diff --git a/families/k2_horizon_uno/support.py b/families/k2_horizon_uno/support.py index c0a7f4d8a4..e4373155da 100644 --- a/families/k2_horizon_uno/support.py +++ b/families/k2_horizon_uno/support.py @@ -7,10 +7,10 @@ _SUPPORT = FamilySupport(tasks=("text_generation",), default_task="text_generation") -_FILES = ("adapter_config.json", "adapter_model.safetensors") +_FILES = frozenset({"adapter_config.json", "adapter_model.safetensors"}) def describe(metadata: ModelMetadata) -> FamilySupport | None: - """Select the two-file adapter snapshot; build validates its exact recipe.""" + """Select snapshots containing the adapter pair; build validates the exact recipe.""" - return _SUPPORT if metadata.files == _FILES else None + return _SUPPORT if _FILES.issubset(metadata.files) else None diff --git a/families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml b/families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml new file mode 100644 index 0000000000..9e10a2074e --- /dev/null +++ b/families/k2_horizon_uno/tests/benchmark/k2-horizon-7b-uno.yaml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: k2-horizon-7b-uno +candidate: + family: k2_horizon_uno + checkpoint: IFM/K2-Horizon-7B-Uno + revision: ec92bbd768f4a404319625204544782e3377bcd7 + task: text_generation + precision: bf16 + trust_remote_code: true + bundle: k2-horizon-7b-uno.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + model: IFM/K2-Horizon-7B + revision: 586b03f0fd1fbbf2f13eeafc33749e95ae34dd10 + trust_remote_code: true + precision: bf16 + request: &speculative-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + generation_mode: linear_spec_lora + block_length: 8 + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-26 + benchmark: text_generation_performance + operation: generate + request: + <<: *speculative-generation + prompt: Reply with the word OK. + max_new_tokens: 26 + use_chat_template: true + enable_thinking: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + model: IFM/K2-Horizon-7B + revision: 586b03f0fd1fbbf2f13eeafc33749e95ae34dd10 + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/k2_horizon_uno/tests/test_model.py b/families/k2_horizon_uno/tests/test_model.py index fcdaf6f911..d1c3ea45dc 100644 --- a/families/k2_horizon_uno/tests/test_model.py +++ b/families/k2_horizon_uno/tests/test_model.py @@ -92,7 +92,12 @@ def test_support_matches_only_the_adapter_identity() -> None: adapter = ModelMetadata({}, {}, ("adapter_config.json", "adapter_model.safetensors")) assert describe(adapter).tasks == ("text_generation",) assert describe(ModelMetadata({}, {}, ("adapter_config.json",))) is None - assert describe(ModelMetadata({}, {}, (*adapter.files, "README.md"))) is None + published = ModelMetadata( + {}, + {}, + (*adapter.files, ".gitattributes", "README.md", "conversion_summary.json"), + ) + assert describe(published).tasks == ("text_generation",) def test_exact_base_config_and_adapter_recipe(tmp_path: Path) -> None: diff --git a/families/lance/tests/benchmark/lance-3b-x2t-image.yaml b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml new file mode 100644 index 0000000000..c43d4da9e9 --- /dev/null +++ b/families/lance/tests/benchmark/lance-3b-x2t-image.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: lance-3b-x2t-image +candidate: {family: lance, checkpoint: bytedance-research/Lance, revision: 7395315758865e6f56ab87ad06a88c7ac172f056, task: vision_language_generation, precision: bf16, bundle: lance-3b-x2t-image.bundle, build: {max_sequence_length: 384}} +reference_environment: + requirements: ../../requirements.txt + build_isolation: false + paths: {reference_repo: trtmc-reference/Lance} +accuracy: + - name: generated-text-parity + benchmark: task_output_parity + operation: generate + request: &request {prompt: "What color is the vehicle in this image? Answer in one word.", image_path: ../data/test_img.jpeg, max_new_tokens: 10} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: bf16, output_contract: normalized-text, adapter_options: {model_subdir: Lance_3B, vit_subdir: Qwen2.5-VL-ViT, resolution: image_768res}} + gate: {max_normalized_edit_distance: 0.15} +performance: + - {name: generate, benchmark: vision_language_pipeline_performance, operation: generate, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: upstream_lance, output_contract: exact-text}} diff --git a/families/lance/tests/benchmark/prepare_environment.py b/families/lance/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..8bbac437dc --- /dev/null +++ b/families/lance/tests/benchmark/prepare_environment.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned family-owned Lance reference checkout.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +REPOSITORY = "https://github.com/bytedance/Lance.git" +REVISION = "4baeee086648996f6ab12e673cbe461b0b149997" + + +def _prepare_image_only_checkout(repository: Path) -> None: + dataset = repository / "data/datasets_custom/validation_dataset.py" + text = dataset.read_text(encoding="utf-8") + for line in ("import decord\n", "from decord import VideoReader\n"): + if line in text: + if text.count(line) != 1: + raise RuntimeError(f"Lance upstream import contract changed: {line.strip()}") + text = text.replace(line, "") + annotation = "video: VideoReader" + if annotation in text: + if text.count(annotation) != 1: + raise RuntimeError("Lance upstream VideoReader annotation contract changed") + text = text.replace(annotation, "video") + dataset.write_text(text, encoding="utf-8") + + +def main() -> None: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "Lance" + if (destination / "inference_lance.py").is_file(): + _prepare_image_only_checkout(destination) + return + if destination.exists(): + raise RuntimeError(f"incomplete Lance reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="Lance.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", REPOSITORY], check=True + ) + subprocess.run( + ["git", "-C", str(temporary), "fetch", "--quiet", "--depth=1", "origin", REVISION], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + _prepare_image_only_checkout(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +if __name__ == "__main__": + main() diff --git a/families/lance/tests/benchmark/reference.py b/families/lance/tests/benchmark/reference.py new file mode 100644 index 0000000000..1dd3d0abf3 --- /dev/null +++ b/families/lance/tests/benchmark/reference.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned Lance reference.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tempfile +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _execute( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Measurement: + reference_repo = Path(str(options.get("reference_repo", ""))).resolve() + if not reference_repo.is_dir(): + raise ValueError("Lance reference requires reference_repo") + if arguments.mode != "pytorch-eager" or arguments.precision != "bf16": + raise ValueError("Lance reference requires pytorch-eager bf16") + with tempfile.TemporaryDirectory(prefix="trtmc-perf-lance-") as temporary: + output = Path(temporary) / "result.json" + command = [ + sys.executable, + str(Path(__file__).with_name("upstream_reference.py")), + "--reference-repo", + str(reference_repo), + "--model", + arguments.model, + "--model-subdir", + str(options.get("model_subdir", "Lance_3B")), + "--vit-subdir", + str(options.get("vit_subdir", "Qwen2.5-VL-ViT")), + "--image", + str(request["image_path"]), + "--prompt", + str(request.get("prompt", "")), + "--instruction", + str(options.get("instruction", "Look at the image carefully and answer the question.")), + "--max-new-tokens", + str(request.get("max_new_tokens", 16)), + "--warmup", + str(arguments.warmup), + "--iterations", + str(arguments.iterations), + "--resolution", + str(options.get("resolution", "image_768res")), + "--height", + str(options.get("height", 768)), + "--width", + str(options.get("width", 768)), + "--output", + str(output), + ] + if arguments.revision: + command.extend(("--revision", arguments.revision)) + if arguments.local_files_only: + command.append("--local-files-only") + completed = subprocess.run( + command, + cwd=reference_repo, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + raise RuntimeError( + f"Lance reference failed with rc={completed.returncode}: {completed.stderr[-2000:]}" + ) + payload = json.loads(output.read_text(encoding="utf-8")) + return reference_harness.Measurement( + [float(value) for value in payload.get("samples_ms", [])], + {"text": str(payload.get("text", "")), "output_tokens": None}, + "lance-pytorch", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run_premeasured( + list(sys.argv[1:] if argv is None else argv), + description=__doc__, + execute=_execute, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/benchmark/performance/baselines/lance_reference.py b/families/lance/tests/benchmark/upstream_reference.py similarity index 100% rename from apps/benchmark/performance/baselines/lance_reference.py rename to families/lance/tests/benchmark/upstream_reference.py diff --git a/families/lance/tests/test_benchmark_reference.py b/families/lance/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..49579f82e2 --- /dev/null +++ b/families/lance/tests/test_benchmark_reference.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + +from families.lance.tests.benchmark import upstream_reference + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_uses_the_family_owned_reference(tmp_path: Path) -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/lance-3b-x2t-image.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert "adapter" not in reference + + checkout = tmp_path / "Lance" + parsed = upstream_reference.build_parser().parse_args( + [ + "--reference-repo", + str(checkout), + "--model", + "model", + "--image", + str(tmp_path / "image.png"), + "--prompt", + "describe", + "--max-new-tokens", + "4", + "--warmup", + "1", + "--iterations", + "2", + "--output", + str(tmp_path / "output.json"), + ] + ) + assert parsed.reference_repo == checkout diff --git a/families/lance/tests/test_official_reference.py b/families/lance/tests/test_official_reference.py index 2cb7ef01cb..30bd39b2cd 100644 --- a/families/lance/tests/test_official_reference.py +++ b/families/lance/tests/test_official_reference.py @@ -8,6 +8,7 @@ from types import SimpleNamespace import pytest +import yaml from families.lance.tests.official_reference import ( _image_only_source, @@ -15,6 +16,15 @@ _source_dir, run_official_generation, ) +from families.lance.tests.benchmark import prepare_environment + + +def test_qualification_environment_keeps_extension_builds_in_family_environment() -> None: + profile = yaml.safe_load( + (Path(__file__).parent / "benchmark/lance-3b-x2t-image.yaml").read_text(encoding="utf-8") + ) + + assert profile["reference_environment"]["build_isolation"] is False def test_reference_source_is_required_and_fail_closed(monkeypatch, tmp_path) -> None: @@ -60,6 +70,21 @@ def test_image_source_removes_only_the_eager_video_import(tmp_path) -> None: assert (image_source / "modeling").is_symlink() +def test_qualification_checkout_removes_only_the_eager_video_import(tmp_path) -> None: + source = tmp_path / "source" + source.mkdir() + _fake_source(source) + + prepare_environment._prepare_image_only_checkout(source) + prepare_environment._prepare_image_only_checkout(source) + + patched = (source / "data/datasets_custom/validation_dataset.py").read_text() + assert "import decord" not in patched + assert "from decord" not in patched + assert "video: VideoReader" not in patched + assert "return VideoReader, decord, video" in patched + + def test_result_requires_one_nonempty_answer(tmp_path) -> None: result = tmp_path / "result.json" result.write_text(json.dumps([{"answer": " White "}]), encoding="utf-8") diff --git a/families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml b/families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml new file mode 100644 index 0000000000..791a7a8d65 --- /dev/null +++ b/families/lerobot_act/tests/benchmark/act-aloha-sim-transfer-cube.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: act-aloha-sim-transfer-cube +candidate: + family: lerobot_act + checkpoint: lerobot/act_aloha_sim_transfer_cube_human + revision: ba73b2766f1371cdc133ca4efb97eb090d744625 + task: robot_control + precision: fp32 + bundle: act-aloha-sim-transfer-cube.bundle + build: + tensor_parallel_size: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: recorded-action-parity + benchmark: robot_action_parity + samples: 1 + reference: + command: reference.py + precision: fp32 + gate: + action_max_abs_error: 0.00005 + action_mean_abs_error: 0.000005 + action_rmse: 0.00001 + min_sample_pass_rate: 1.0 + +performance: + - name: control + benchmark: robot_action_performance + operation: control + request: + image_path: ../data/recorded_observation/observation.images.top.png + state_path: ../data/recorded_observation/observation.state.f32 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: official_lerobot + precision: fp32 + output_contract: robot-action-parity + action_max_abs_error: 0.00005 + action_mean_abs_error: 0.000005 + action_rmse: 0.00001 diff --git a/families/lerobot_act/tests/benchmark/prepare_environment.py b/families/lerobot_act/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4874c92184 --- /dev/null +++ b/families/lerobot_act/tests/benchmark/prepare_environment.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned LeRobot source inside this family's environment.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +SOURCE_REPOSITORY = "https://github.com/huggingface/lerobot.git" +SOURCE_REVISION = "3c0a209f9fac4d2a57617e686a7f2a2309144ba2" + + +def main() -> int: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "lerobot" + required = destination / "lerobot/common/policies/act/modeling_act.py" + if required.is_file(): + return 0 + if destination.exists(): + raise RuntimeError(f"incomplete LeRobot reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="lerobot.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", SOURCE_REPOSITORY], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(temporary), + "fetch", + "--quiet", + "--depth=1", + "origin", + SOURCE_REVISION, + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/lerobot_act/tests/benchmark/reference.py b/families/lerobot_act/tests/benchmark/reference.py new file mode 100644 index 0000000000..567b48ab2e --- /dev/null +++ b/families/lerobot_act/tests/benchmark/reference.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned official Accuracy and Performance reference for LeRobot ACT.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import statistics +import sys +import time +from typing import Any, Mapping + +import numpy as np + +from families.lerobot_act.tests.official_reference import ( + load_observation, + load_policy, + predict_actions, +) + + +def _source_root() -> Path: + root = Path(sys.prefix) / "trtmc-reference/lerobot" + if not (root / "lerobot/common/policies/act/modeling_act.py").is_file(): + raise RuntimeError(f"LeRobot reference is incomplete: {root}") + return root + + +def _checkpoint(model: str, revision: str | None, local_files_only: bool) -> Path: + path = Path(model).expanduser() + if path.is_dir(): + return path.resolve() + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + allow_patterns=("config.json", "model.safetensors"), + local_files_only=local_files_only, + ) + ) + + +def _summary(actions: np.ndarray) -> dict[str, Any]: + return { + "action_steps": int(actions.shape[0]), + "action_dim": int(actions.shape[1]), + "action_values": int(actions.size), + "actions": actions.reshape(-1).tolist(), + "finite": bool(np.isfinite(actions).all()), + } + + +def _run_accuracy(arguments: argparse.Namespace) -> int: + payload = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = payload.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("Accuracy reference requires samples") + checkpoint = _checkpoint( + str(payload["model"]), + payload.get("revision"), + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + ) + torch, policy, config, device = load_policy(_source_root(), checkpoint) + output_samples = [] + for index, sample in enumerate(samples): + if not isinstance(sample, Mapping): + raise ValueError(f"Accuracy sample {index} must be an object") + pixels, state = load_observation( + Path(str(sample["image_path"])), Path(str(sample["state_path"])) + ) + actions = predict_actions(torch, policy, config, device, pixels, state) + output_samples.append( + {"sample_id": str(sample["sample_id"]), **_summary(actions)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": output_samples}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +def _run_performance(arguments: argparse.Namespace) -> int: + import torch + + request = json.loads(arguments.request_json) + timing = json.loads(arguments.timing_contract_json) + checkpoint = _checkpoint(arguments.model, arguments.revision, arguments.local_files_only) + _, policy, config, device = load_policy(_source_root(), checkpoint) + pixels, state = load_observation( + Path(str(request["image_path"])), Path(str(request["state_path"])) + ) + + def invoke(): + return predict_actions(torch, policy, config, device, pixels, state) + + actions = None + for _ in range(arguments.warmup): + actions = invoke() + samples_ms = [] + for _ in range(arguments.iterations): + if torch.cuda.is_available(): + torch.cuda.synchronize() + started = time.perf_counter_ns() + actions = invoke() + if torch.cuda.is_available(): + torch.cuda.synchronize() + samples_ms.append((time.perf_counter_ns() - started) / 1_000_000) + if actions is None: + raise RuntimeError("LeRobot reference produced no actions") + value = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + "framework": f"lerobot-torch-{torch.__version__}", + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": timing, + **timing, + "samples_ms": samples_ms, + "metrics": {"latency_ms": {"p50": statistics.median(samples_ms)}}, + "output_summary": _summary(actions), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + for name in ( + "model", + "family", + "manifest", + "operation", + "selected-task", + "request-json", + "adapter-options-json", + "timing-contract-json", + "precision", + "mode", + "padding", + "case-name", + ): + parser.add_argument("--" + name, required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + return parser + + +def main() -> int: + if "--request" in sys.argv[1:]: + return _run_accuracy(_accuracy_parser().parse_args()) + return _run_performance(_performance_parser().parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/lerobot_act/tests/data/benchmark_robot_control.json b/families/lerobot_act/tests/data/benchmark_robot_control.json new file mode 100644 index 0000000000..3ece00d678 --- /dev/null +++ b/families/lerobot_act/tests/data/benchmark_robot_control.json @@ -0,0 +1,9 @@ +{ + "requests": [ + { + "id": "aloha-sim-transfer-cube-episode-0-frame-0", + "image": "recorded_observation/observation.images.top.png", + "state": "recorded_observation/observation.state.f32" + } + ] +} diff --git a/families/lerobot_act/tests/test_benchmark_reference.py b/families/lerobot_act/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..8d92ca6849 --- /dev/null +++ b/families/lerobot_act/tests/test_benchmark_reference.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/act-aloha-sim-transfer-cube.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert reference["output_contract"] == "robot-action-parity" + assert "adapter" not in reference diff --git a/families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml b/families/lfm2/tests/benchmark/lfm2-1.2b.yaml similarity index 80% rename from families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml rename to families/lfm2/tests/benchmark/lfm2-1.2b.yaml index 3a9007aee9..13811fc77a 100644 --- a/families/gpt_neo/tests/benchmark/gpt-neo-125m.yaml +++ b/families/lfm2/tests/benchmark/lfm2-1.2b.yaml @@ -2,15 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: gpt-neo-125m +model: lfm2-1.2b candidate: - family: gpt_neo - checkpoint: EleutherAI/gpt-neo-125m + family: lfm2 + checkpoint: LiquidAI/LFM2-1.2B + revision: 40f3da0d0164913923aee9462c23077868b816a3 task: text_generation precision: fp16 - bundle: gpt-neo-125m.bundle + bundle: lfm2-1.2b-fp16.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 accuracy: - name: mmlu-continuation @@ -19,7 +20,7 @@ accuracy: prompt_token_limit: 192 truncation_side: left reference: - precision: fp32 + precision: bf16 request: &greedy-generation max_new_tokens: 64 temperature: 0.0 @@ -36,12 +37,13 @@ accuracy: allowed_failures: 1 performance: - - name: generate-64 + - name: generate-8 benchmark: text_generation_performance operation: generate request: <<: *greedy-generation prompt: The capital of France is + max_new_tokens: 8 measurement: warmup: 5 iterations: 10 @@ -50,5 +52,5 @@ performance: mode: torch-compile fallback: hf-eager compile_scope: model.forward - precision: fp32 + precision: bf16 output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-2.6b.yaml b/families/lfm2/tests/benchmark/lfm2-2.6b.yaml new file mode 100644 index 0000000000..cc0ce20ae6 --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-2.6b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-2.6b +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-2.6B + revision: 36ed799f7024ef169ba92ffe184821835e15cc66 + task: text_generation + precision: fp16 + bundle: lfm2-2.6b-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/bloom/tests/benchmark/bloom-560m.yaml b/families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml similarity index 77% rename from families/bloom/tests/benchmark/bloom-560m.yaml rename to families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml index eb8482c31b..d7dd7abc68 100644 --- a/families/bloom/tests/benchmark/bloom-560m.yaml +++ b/families/lfm2/tests/benchmark/lfm2-350m-bf16-model-card.yaml @@ -2,15 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: bloom-560m +model: lfm2-350m-bf16-model-card candidate: - family: bloom - checkpoint: bigscience/bloom-560m + family: lfm2 + checkpoint: LiquidAI/LFM2-350M + revision: f37d3f5c8c5484bc01dad379a595cf4c68c4e70e task: text_generation - precision: fp32 - bundle: bloom-560m.bundle + precision: bf16 + bundle: lfm2-350m-bf16-model-card.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 accuracy: - name: mmlu-continuation @@ -19,7 +20,7 @@ accuracy: prompt_token_limit: 192 truncation_side: left reference: - precision: fp32 + precision: bf16 request: &greedy-generation max_new_tokens: 64 temperature: 0.0 @@ -36,12 +37,13 @@ accuracy: allowed_failures: 1 performance: - - name: generate-64 + - name: generate-12 benchmark: text_generation_performance operation: generate request: <<: *greedy-generation prompt: The capital of France is + max_new_tokens: 12 measurement: warmup: 5 iterations: 10 @@ -50,5 +52,5 @@ performance: mode: torch-compile fallback: hf-eager compile_scope: model.forward - precision: fp32 + precision: bf16 output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml b/families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml new file mode 100644 index 0000000000..301da1c28e --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-350m-fp16.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-350m-fp16 +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-350M + revision: f37d3f5c8c5484bc01dad379a595cf4c68c4e70e + task: text_generation + precision: fp16 + bundle: lfm2-350m-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-12 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 12 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/lfm2/tests/benchmark/lfm2-700m.yaml b/families/lfm2/tests/benchmark/lfm2-700m.yaml new file mode 100644 index 0000000000..e3e797cc70 --- /dev/null +++ b/families/lfm2/tests/benchmark/lfm2-700m.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: lfm2-700m +candidate: + family: lfm2 + checkpoint: LiquidAI/LFM2-700M + revision: 86f49fc9a3800c3a325b7320bde179c318062583 + task: text_generation + precision: fp16 + bundle: lfm2-700m-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/llama/runtime/bpe_tokenizer.cpp b/families/llama/runtime/bpe_tokenizer.cpp index 53a7a3ab2a..860defc1df 100644 --- a/families/llama/runtime/bpe_tokenizer.cpp +++ b/families/llama/runtime/bpe_tokenizer.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,54 @@ inline std::string utf32_to_utf8(char32_t cp) { return r; } +std::string replace_invalid_utf8(const std::string& text) { + static constexpr std::string_view replacement = "\xef\xbf\xbd"; + std::string result; + result.reserve(text.size()); + std::size_t pos = 0; + while (pos < text.size()) { + const auto lead = static_cast(text[pos]); + if (lead < 0x80) { + result.push_back(text[pos++]); + continue; + } + + std::size_t length = 0; + if (lead >= 0xC2 && lead <= 0xDF) + length = 2; + else if (lead >= 0xE0 && lead <= 0xEF) + length = 3; + else if (lead >= 0xF0 && lead <= 0xF4) + length = 4; + else { + result.append(replacement); + ++pos; + continue; + } + + if (pos + length > text.size()) { + result.append(replacement); + break; + } + const auto second = static_cast(text[pos + 1]); + bool valid = (second & 0xC0) == 0x80; + if (length >= 3) + valid = valid && (static_cast(text[pos + 2]) & 0xC0) == 0x80; + if (length == 4) + valid = valid && (static_cast(text[pos + 3]) & 0xC0) == 0x80; + valid = valid && !(lead == 0xE0 && second < 0xA0) && !(lead == 0xED && second >= 0xA0) && + !(lead == 0xF0 && second < 0x90) && !(lead == 0xF4 && second >= 0x90); + if (!valid) { + result.append(replacement); + ++pos; + continue; + } + result.append(text, pos, length); + pos += length; + } + return result; +} + // Read one UTF-8 codepoint from raw bytes, advance ptr. Returns 0xFFFD on error. inline char32_t read_utf8(const char*& p, const char* end) { if (p >= end) @@ -1009,7 +1058,7 @@ class BpeTokenizer final : public ITokenizer { result.append(text, start, pos - start); } } - return result; + return replace_invalid_utf8(result); } // Generic pre-tokenizer used when no GPT-2 pattern is declared. diff --git a/families/granite/tests/benchmark/granite-3.1-2b.yaml b/families/llama/tests/benchmark/falcon3-1b.yaml similarity index 83% rename from families/granite/tests/benchmark/granite-3.1-2b.yaml rename to families/llama/tests/benchmark/falcon3-1b.yaml index 15e351fe3d..66d7e02d18 100644 --- a/families/granite/tests/benchmark/granite-3.1-2b.yaml +++ b/families/llama/tests/benchmark/falcon3-1b.yaml @@ -2,13 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: granite-3.1-2b +model: falcon3-1b candidate: - family: granite - checkpoint: ibm-granite/granite-3.1-2b-base + family: llama + checkpoint: tiiuae/Falcon3-1B-Base + revision: cb37ef3559b157b5c9d9226296ba01a5162da1f7 task: text_generation precision: fp16 - bundle: granite-3.1-2b.bundle + bundle: falcon3-1b.bundle build: max_sequence_length: 256 @@ -41,7 +42,7 @@ performance: operation: generate request: <<: *greedy-generation - prompt: The capital of France is + prompt: The quick brown fox jumps over the lazy dog. Once upon a time max_new_tokens: 20 measurement: warmup: 5 diff --git a/families/llama/tests/benchmark/tinyllama-1.1b.yaml b/families/llama/tests/benchmark/minicpm5-2b.yaml similarity index 88% rename from families/llama/tests/benchmark/tinyllama-1.1b.yaml rename to families/llama/tests/benchmark/minicpm5-2b.yaml index b29c87ce16..b5cf7288bd 100644 --- a/families/llama/tests/benchmark/tinyllama-1.1b.yaml +++ b/families/llama/tests/benchmark/minicpm5-2b.yaml @@ -2,15 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: tinyllama-1.1b +model: minicpm5-2b candidate: family: llama - checkpoint: TinyLlama/TinyLlama-1.1B-Chat-v1.0 + checkpoint: openbmb/MiniCPM5-2B + revision: 12a3808a956f869c767195e9266b59c4d21d92e2 task: text_generation precision: fp16 - bundle: tinyllama-1.1b.bundle + bundle: minicpm5-2b.bundle build: - max_sequence_length: 258 + max_sequence_length: 257 accuracy: - name: mmlu-continuation diff --git a/families/llama/tests/benchmark/minitron-4b-depth.yaml b/families/llama/tests/benchmark/minitron-4b-depth.yaml new file mode 100644 index 0000000000..cbb5ee26a0 --- /dev/null +++ b/families/llama/tests/benchmark/minitron-4b-depth.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: minitron-4b-depth +candidate: + family: llama + checkpoint: nvidia/Llama-3.1-Minitron-4B-Depth-Base + revision: 47f17b2c73bbd07a4b643a0382970704d389f737 + task: text_generation + precision: fp16 + bundle: minitron-4b-depth.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. Once upon a time + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/llama/tests/benchmark/minitron-4b-width.yaml b/families/llama/tests/benchmark/minitron-4b-width.yaml new file mode 100644 index 0000000000..af9864c8a5 --- /dev/null +++ b/families/llama/tests/benchmark/minitron-4b-width.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: minitron-4b-width +candidate: + family: llama + checkpoint: nvidia/Llama-3.1-Minitron-4B-Width-Base + revision: 5205ef7d36204947e3b973cb8b147a816ccd7e6a + task: text_generation + precision: fp16 + bundle: minitron-4b-width.bundle + build: + max_sequence_length: 131072 + dynamic_kv_cache: true + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The quick brown fox jumps over the lazy dog. Once upon a time + max_new_tokens: 20 + kv_cache_size: 1GiB + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/llama/tests/benchmark/nemotron-nano-4b.yaml b/families/llama/tests/benchmark/nemotron-nano-4b.yaml new file mode 100644 index 0000000000..5c27961e07 --- /dev/null +++ b/families/llama/tests/benchmark/nemotron-nano-4b.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-nano-4b +candidate: + family: llama + checkpoint: nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1 + revision: d552708a9d575fa8d4a690b988fd870d65279f98 + task: text_generation + precision: fp16 + bundle: nemotron-nano-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? + max_new_tokens: 20 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/llama/tests/cpp/test_llama_plugin_helpers.cpp b/families/llama/tests/cpp/test_llama_plugin_helpers.cpp index 8ed30ed094..f218fdfa3f 100644 --- a/families/llama/tests/cpp/test_llama_plugin_helpers.cpp +++ b/families/llama/tests/cpp/test_llama_plugin_helpers.cpp @@ -58,6 +58,18 @@ std::string tokenizer_json() { })"; } +std::string byte_level_tokenizer_json() { + return R"({ + "model": { + "type": "BPE", + "unk_token": "", + "vocab": {"": 0, "\u00e2": 1, "\u012b": 2, "\u00a5": 3}, + "merges": [] + }, + "decoder": {"type": "ByteLevel"} + })"; +} + std::filesystem::path write_tokenizer_bundle(std::string_view tokenizer) { char path[] = "/tmp/trtmc_llama_plugin_helpers_XXXXXX"; const int descriptor = mkstemp(path); @@ -103,11 +115,21 @@ void test_exact_special_frame_respects_add_special_false() { "exact post-processor frame respects add_special_tokens=false"); } +void test_byte_level_decode_replaces_incomplete_utf8() { + const std::string json = byte_level_tokenizer_json(); + const auto tokenizer = trtmc::CreateBpeTokenizer(json.data(), json.size(), false); + check(tokenizer->decode({1, 2}) == "\xef\xbf\xbd", + "byte-level decode replaces an incomplete UTF-8 suffix"); + check(tokenizer->decode({1, 2, 3}) == "\xe2\x89\xa5", + "byte-level decode preserves a complete UTF-8 sequence"); +} + } // namespace int main() { test_exact_special_frame_uses_bundle_tokenizer(); test_exact_special_frame_respects_add_special_false(); + test_byte_level_decode_replaces_incomplete_utf8(); if (failures != 0) std::cerr << failures << " Llama plugin-helper test(s) failed\n"; return failures; diff --git a/families/locateanything/model.py b/families/locateanything/model.py index 144218cd7b..a0917a6cf0 100644 --- a/families/locateanything/model.py +++ b/families/locateanything/model.py @@ -338,15 +338,11 @@ def _detect_layer_prefix(readers) -> str: def _tokenizer_runtime_contract(model_dir: Path) -> dict[str, object]: """Resolve this family's exact native-tokenizer framing.""" - from transformers import AutoTokenizer + from tokenizers import Tokenizer - tokenizer = AutoTokenizer.from_pretrained( - str(model_dir), - trust_remote_code=True, - use_fast=True, - ) - default_ids = list(tokenizer.encode("hello")) - plain_ids = list(tokenizer.encode("hello", add_special_tokens=False)) + tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json")) + default_ids = list(tokenizer.encode("hello").ids) + plain_ids = list(tokenizer.encode("hello", add_special_tokens=False).ids) if default_ids == plain_ids: prefix_ids, suffix_ids = [], [] elif not plain_ids: diff --git a/families/locateanything/tests/benchmark/locateanything-3b.yaml b/families/locateanything/tests/benchmark/locateanything-3b.yaml new file mode 100644 index 0000000000..88a361eacb --- /dev/null +++ b/families/locateanything/tests/benchmark/locateanything-3b.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: locateanything-3b +candidate: + family: locateanything + checkpoint: nvidia/LocateAnything-3B + revision: c32291ca5e996f5a7a485845b4f57a233936bba0 + task: vision_language_generation + precision: fp16 + trust_remote_code: true + bundle: locateanything-3b.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-localization-parity + benchmark: imagenette_localization_parity + samples: 10 + request: + prompt_template: "Locate a single instance that matches the following description: {label}." + max_new_tokens: 32 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + min_localization_box_iou: 0.9 + max_localization_point_distance: 10.0 + max_normalized_edit_distance: 0.5 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + image_path: ../data/test_img.jpeg + prompt: "Locate a single instance that matches the following description: white vehicle." + max_new_tokens: 32 + temperature: 1.0 + top_k: 1 + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: localization + min_localization_box_iou: 0.9 + max_localization_point_distance: 10.0 + max_normalized_edit_distance: 0.5 diff --git a/families/locateanything/tests/benchmark/reference.py b/families/locateanything/tests/benchmark/reference.py new file mode 100644 index 0000000000..d4c9a9d06a --- /dev/null +++ b/families/locateanything/tests/benchmark/reference.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official LocateAnything reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + +from families.locateanything.tests.hf_reference import ( + _LocalTokenizer, + _load_config, + _repair_rotary_buffers, + manual_chat_prompt, +) +from families.locateanything.tests.vision_oracle import preprocess_image_inputs_for_trt + + +def _model_directory(model: str, revision: str | None) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ), + ) + ) + + +def _infer(model, tokenizer, image_path: str, prompt: str, max_new_tokens: int) -> str: + import torch + + image_inputs = preprocess_image_inputs_for_trt( + Path(image_path), + fixed_image_size=448, + patch_size=14, + image_mean=(0.5, 0.5, 0.5), + image_std=(0.5, 0.5, 0.5), + interpolation="bicubic", + ) + pixel_values = torch.from_numpy(image_inputs["pixel_values"]).to("cuda") + image_grid_hws = torch.from_numpy(image_inputs["image_grid_hws"]).to( + device="cuda", dtype=torch.int32 + ) + inputs = tokenizer(manual_chat_prompt(prompt), return_tensors="pt") + input_ids = inputs["input_ids"].to("cuda") + attention_mask = inputs["attention_mask"].to("cuda") + with torch.inference_mode(): + output = model.generate( + pixel_values=pixel_values, + image_grid_hws=image_grid_hws, + input_ids=input_ids, + attention_mask=attention_mask, + tokenizer=tokenizer, + max_new_tokens=max_new_tokens, + use_cache=True, + generation_mode="slow", + do_sample=False, + ) + if isinstance(output, str): + text = output + elif isinstance(output, (list, tuple)) and output and isinstance(output[0], str): + text = output[0] + else: + token_ids = output[0] if output.ndim > 1 else output + if token_ids.numel() > input_ids.shape[-1]: + token_ids = token_ids[input_ids.shape[-1] :] + text = tokenizer.decode(token_ids, skip_special_tokens=True) + return str(text).strip() + + +def _run_accuracy(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModel + + if not torch.cuda.is_available(): + raise RuntimeError("LocateAnything Accuracy reference requires CUDA") + if request.get("precision") != "fp32": + raise ValueError("LocateAnything official reference requires fp32") + model_dir = _model_directory(str(request["model"]), request.get("revision")) + config = _load_config(model_dir) + tokenizer = _LocalTokenizer(model_dir) + model = ( + AutoModel.from_pretrained( + model_dir, + config=config, + trust_remote_code=True, + local_files_only=True, + torch_dtype=torch.float32, + ) + .to("cuda") + .eval() + ) + _repair_rotary_buffers(model) + controls = request.get("request", {}) + prompt_template = controls.get("prompt_template") if isinstance(controls, dict) else None + if not isinstance(prompt_template, str) or "{label}" not in prompt_template: + raise ValueError("reference request.prompt_template must contain {label}") + max_new_tokens = int(controls.get("max_new_tokens", 32)) + results = [] + for sample in request.get("samples", []): + prompt = prompt_template.format(label=sample["label_name"]) + text = _infer(model, tokenizer, str(sample["image_path"]), prompt, max_new_tokens) + if not text: + raise RuntimeError("LocateAnything reference produced empty text") + results.append({"sample_id": str(sample["sample_id"]), "text": text}) + if not results: + raise ValueError("reference request must contain samples") + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _load_performance( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from transformers import AutoModel + + if arguments.precision != "fp32": + raise ValueError("LocateAnything official reference requires fp32") + model_dir = _model_directory(arguments.model, arguments.revision) + config = _load_config(model_dir) + tokenizer = _LocalTokenizer(model_dir) + model = ( + AutoModel.from_pretrained( + model_dir, + config=config, + trust_remote_code=arguments.trust_remote_code, + local_files_only=arguments.local_files_only, + torch_dtype=torch.float32, + ) + .to("cuda") + .eval() + ) + _repair_rotary_buffers(model) + image_path = str(request["image_path"]) + prompt = str(request.get("prompt", "")) + max_new_tokens = int(request.get("max_new_tokens", 32)) + + def invoke() -> Mapping[str, Any]: + text = _infer(model, tokenizer, image_path, prompt, max_new_tokens) + if not text: + raise RuntimeError("LocateAnything reference produced empty text") + token_ids = tokenizer.encode(text, add_special_tokens=False) + return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if "--request" in arguments: + return _run_accuracy(arguments) + return reference_harness.run(arguments, description=__doc__, load=_load_performance) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/locateanything/tests/test_benchmark_reference.py b/families/locateanything/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..0597913311 --- /dev/null +++ b/families/locateanything/tests/test_benchmark_reference.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/locateanything-3b.yaml").read_text(encoding="utf-8") + ) + reference = profile["performance"][0]["reference"] + assert reference["script"] == "tests/benchmark/reference.py" + assert reference["output_contract"] == "localization" + assert "adapter" not in reference diff --git a/families/locateanything/tests/test_hf_reference.py b/families/locateanything/tests/test_hf_reference.py index e8b24c8c04..0630d217a4 100644 --- a/families/locateanything/tests/test_hf_reference.py +++ b/families/locateanything/tests/test_hf_reference.py @@ -4,8 +4,11 @@ from __future__ import annotations import inspect +import sys from pathlib import Path +from types import SimpleNamespace +from families.locateanything import model from families.locateanything.tests import hf_reference, vision_oracle @@ -45,3 +48,30 @@ def test_hf_reference_is_local_manual_and_slow() -> None: assert 'generation_mode="slow"' in source assert "preprocess_image_inputs_for_trt(" in source assert "local_files_only=True" in source + + +def test_builder_uses_tokenizer_json_for_native_framing(monkeypatch, tmp_path) -> None: + calls: list[str] = [] + + class Encoding: + def __init__(self, ids: list[int]) -> None: + self.ids = ids + + class LocalTokenizer: + def encode(self, _text: str, add_special_tokens: bool = True) -> Encoding: + return Encoding([1, 7, 2] if add_special_tokens else [7]) + + class Tokenizer: + @staticmethod + def from_file(path: str) -> LocalTokenizer: + calls.append(path) + return LocalTokenizer() + + monkeypatch.setitem(sys.modules, "tokenizers", SimpleNamespace(Tokenizer=Tokenizer)) + + assert model._tokenizer_runtime_contract(tmp_path) == { + "tokenizer_add_special_tokens": False, + "tokenizer_prefix_ids": [1], + "tokenizer_suffix_ids": [2], + } + assert calls == [str(tmp_path / "tokenizer.json")] diff --git a/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml new file mode 100644 index 0000000000..bdae110537 --- /dev/null +++ b/families/m2m_100/tests/benchmark/nllb-200-distilled-600m.yaml @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nllb-200-distilled-600m +candidate: + family: m2m_100 + checkpoint: facebook/nllb-200-distilled-600M + revision: f8d333a098d19b4fd9a8b18f94170487ad3f821d + task: text_generation + precision: fp16 + bundle: nllb-200-distilled-600m.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: flores200-en-fr + benchmark: flores200_en_fr_seq2seq + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp16 + task: seq2seq-lm + output_token_policy: strip-start-and-eos + source_language_placement: replace-final-unk + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + source_language: eng_Latn + source_language_token_id: 256047 + target_language: fra_Latn + forced_bos_token_id: 256057 + gate: + min_pass_rate: 1.0 + allowed_failures: 0 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: The house is wonderful. + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start-and-eos + source_language_placement: replace-final-unk + precision: fp16 + output_contract: exact-token-ids diff --git a/families/m2m_100/tests/test_translation_contract.py b/families/m2m_100/tests/test_translation_contract.py index 467bb42183..82e75af64d 100644 --- a/families/m2m_100/tests/test_translation_contract.py +++ b/families/m2m_100/tests/test_translation_contract.py @@ -4,10 +4,24 @@ import json from pathlib import Path +import yaml + TEST_ROOT = Path(__file__).resolve().parent +def test_qualification_profile_owns_source_language_placement() -> None: + profile = yaml.safe_load( + (TEST_ROOT / "benchmark/nllb-200-distilled-600m.yaml").read_text(encoding="utf-8") + ) + + cases = [*profile["accuracy"], *profile["performance"]] + assert cases + assert all( + case["reference"]["source_language_placement"] == "replace-final-unk" for case in cases + ) + + def test_nllb_case_keeps_the_explicit_english_to_french_contract() -> None: manifest = json.loads((TEST_ROOT / "manifests/nllb-200.json").read_text(encoding="utf-8")) case = manifest["testcases"][0] diff --git a/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml b/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml new file mode 100644 index 0000000000..1b657ddf26 --- /dev/null +++ b/families/magpie_tts/tests/benchmark/magpie-tts-357m.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: magpie-tts-357m +candidate: {family: magpie_tts, checkpoint: nvidia/magpie_tts_multilingual_357m, revision: 34d7e40da85cabc97f92198889b65cea27bc7fd1, task: audio_generation, precision: fp32, bundle: magpie-tts-357m.bundle, build: {max_sequence_length: 512}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-audio-parity + benchmark: task_output_parity + operation: generate_audio + request: &request {prompt: Please say this sentence clearly., max_new_tokens: 260, seed: 42} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: fp32, output_contract: audio-artifact-parity, adapter_options: {speaker_encoder_revision: e9124b5364a2c3e9b4f78da429a33cbca8f8c22b}} + gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} +performance: + - {name: generate-audio, benchmark: audio_generation_pipeline_performance, operation: generate_audio, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: nemo_reference, output_contract: audio-shape}} diff --git a/families/magpie_tts/tests/benchmark/reference.py b/families/magpie_tts/tests/benchmark/reference.py new file mode 100644 index 0000000000..f9bc1efb41 --- /dev/null +++ b/families/magpie_tts/tests/benchmark/reference.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned Magpie TTS reference.""" + +from __future__ import annotations + +import os +from pathlib import Path +import random +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +SPEAKER_REPOSITORY = "Edresson/Speaker_Encoder_H_ASP" +SPEAKER_FILENAME = "pytorch_model.bin" +SPEAKER_URL = "https://huggingface.co/Edresson/Speaker_Encoder_H_ASP/resolve/main/pytorch_model.bin" + + +def _seed(torch: Any, value: int) -> None: + import numpy as np + + random.seed(value) + np.random.seed(value) + torch.manual_seed(value) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(value) + + +def _materialize(summary: dict[str, Any], output: Path) -> None: + import soundfile as sound + + audio = summary.pop("_audio_f32") + artifact = output.with_suffix(".audio.wav").resolve() + artifact.parent.mkdir(parents=True, exist_ok=True) + sound.write(artifact, audio, int(summary["sample_rate"]), subtype="FLOAT") + summary["audio_artifact"] = str(artifact) + + +def _load( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager" or arguments.precision != "fp32": + raise ValueError("Magpie TTS reference requires pytorch-eager fp32") + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + import fsspec + import torch + from huggingface_hub import hf_hub_download + from nemo.collections.tts.models import MagpieTTSModel + + torch.use_deterministic_algorithms(True) + speaker_revision = str(options.get("speaker_encoder_revision", "") or "") + if not speaker_revision: + raise ValueError("Magpie TTS requires speaker_encoder_revision") + speaker_checkpoint = hf_hub_download( + repo_id=SPEAKER_REPOSITORY, + filename=SPEAKER_FILENAME, + revision=speaker_revision, + local_files_only=arguments.local_files_only, + ) + original_open = fsspec.open + + def offline_open(path: Any, *args: Any, **kwargs: Any) -> Any: + if str(path).split("?", 1)[0] == SPEAKER_URL: + path = speaker_checkpoint + return original_open(path, *args, **kwargs) + + fsspec.open = offline_open + archive = hf_hub_download( + repo_id=arguments.model, + filename="magpie_tts_multilingual_357m.nemo", + revision=arguments.revision, + local_files_only=arguments.local_files_only, + ) + model = MagpieTTSModel.restore_from(restore_path=archive).eval().to("cuda") + max_new_tokens = request.get("max_new_tokens", 0) + if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): + raise ValueError("Magpie max_new_tokens must be an integer") + if max_new_tokens > 0: + model.inference_parameters.max_decoder_steps = max_new_tokens + seed = request.get("seed", 42) + if isinstance(seed, bool) or not isinstance(seed, int): + raise ValueError("Magpie seed must be an integer") + prompt = str(request.get("prompt", "")) + + def invoke() -> Mapping[str, Any]: + _seed(torch, seed) + with torch.inference_mode(): + audio, length = model.do_tts(transcript=prompt, language="en", use_cfg=True) + count = int(length.item()) if length.numel() else int(audio.numel()) + return { + "audio_samples": count, + "sample_rate": 22_050, + "_audio_f32": audio.detach().float().cpu().reshape(-1)[:count].numpy(), + } + + return reference_harness.Session( + invoke, + "nemo", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + materialize=_materialize, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run( + list(sys.argv[1:] if argv is None else argv), + description=__doc__, + load=_load, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/marian/tests/benchmark/marian-en-ru.yaml b/families/marian/tests/benchmark/marian-en-ru.yaml new file mode 100644 index 0000000000..c5e1748028 --- /dev/null +++ b/families/marian/tests/benchmark/marian-en-ru.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: marian-en-ru +candidate: + family: marian + checkpoint: Helsinki-NLP/opus-mt-en-ru + revision: bb09c99d180016eac6819df3dae68edb1690fdee + task: text_generation + precision: fp16 + bundle: marian-en-ru.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: newstest2019-en-ru + benchmark: newstest2019_en_ru_translation + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp32 + task: seq2seq-lm + output_token_policy: strip-start + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: The house is wonderful. + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start + precision: fp32 + output_contract: exact-token-ids diff --git a/families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml b/families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml new file mode 100644 index 0000000000..2a916c72d5 --- /dev/null +++ b/families/minimax_h3/tests/benchmark/minimax-h3-768p.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: minimax-h3-768p +candidate: + family: minimax_h3 + checkpoint: MiniMaxAI/MiniMax-H3 + revision: 48d93ede732756e404a3b1b2f3b3a9b5a22f6cfc + task: image_generation + precision: bf16 + bundle: minimax-h3-768p.bundle + build: {image_height: 768, image_width: 1344, video_num_frames: 124} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt_path: ../prompts/t2va-example-1.json, media_type: video, seed: 0, num_steps: 50, height: 768, width: 1344, video_num_frames: 124} + reference: &reference {adapter: hf-diffusers-modular, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {workflow: t2va, output: videos}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/rwkv/tests/benchmark/rwkv-169m.yaml b/families/mistral/tests/benchmark/mistral-7b.yaml similarity index 78% rename from families/rwkv/tests/benchmark/rwkv-169m.yaml rename to families/mistral/tests/benchmark/mistral-7b.yaml index 39942ec515..8f29cf0736 100644 --- a/families/rwkv/tests/benchmark/rwkv-169m.yaml +++ b/families/mistral/tests/benchmark/mistral-7b.yaml @@ -2,15 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: rwkv-169m +model: mistral-7b candidate: - family: rwkv - checkpoint: RWKV/rwkv-4-169m-pile + family: mistral + checkpoint: mistralai/Mistral-7B-Instruct-v0.1 + revision: ec5deb64f2c6e6fa90c1abf74a91d5c93a9669ca task: text_generation - precision: fp32 - bundle: rwkv-169m.bundle + precision: fp16 + bundle: mistral-7b.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 accuracy: - name: mmlu-continuation @@ -41,8 +42,9 @@ performance: operation: generate request: <<: *greedy-generation - prompt: The capital of France is + prompt: What is the capital of Japan? Answer in one word. max_new_tokens: 10 + use_chat_template: true measurement: warmup: 5 iterations: 10 diff --git a/families/mistral/tests/benchmark/riva-translate-4b.yaml b/families/mistral/tests/benchmark/riva-translate-4b.yaml new file mode 100644 index 0000000000..8b2f8e2559 --- /dev/null +++ b/families/mistral/tests/benchmark/riva-translate-4b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: riva-translate-4b +candidate: + family: mistral + checkpoint: nvidia/Riva-Translate-4B-Instruct-v1.1 + revision: 1eb944ce9f2717d7be6b7b7c2310f7c58b35d10b + task: text_generation + precision: fp16 + bundle: riva-translate-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: flores200-en-fr + benchmark: flores200_en_fr_translation + samples: 10 + prompt_token_limit: 128 + truncation_side: right + reference: + precision: fp32 + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: "System\nYou are an expert at translating text from English to French.\nUser\nWhat is the French translation of the sentence: Hello, how are you?\nAssistant\n" + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/modernbert/tests/benchmark/modernbert-base.yaml b/families/modernbert/tests/benchmark/modernbert-base.yaml new file mode 100644 index 0000000000..1b38bebee2 --- /dev/null +++ b/families/modernbert/tests/benchmark/modernbert-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: modernbert-base +candidate: + family: modernbert + checkpoint: answerdotai/ModernBERT-base + revision: 8949b909ec900327062f0ebf497f51aef5e6f0c8 + task: encoding + precision: fp32 + bundle: modernbert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.995 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/moge/tests/benchmark/moge-2-vitl.yaml b/families/moge/tests/benchmark/moge-2-vitl.yaml new file mode 100644 index 0000000000..f38c24b510 --- /dev/null +++ b/families/moge/tests/benchmark/moge-2-vitl.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: moge-2-vitl +candidate: + family: moge + checkpoint: Ruicheng/moge-2-vitl + revision: 39c4d5e957afe587e04eec59dc2bcc3be5ecd968 + task: monocular_geometry + precision: fp32 + bundle: moge-2-vitl.bundle + build: + tensor_parallel_size: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: metric-geometry-parity + benchmark: metric_geometry_parity + samples: 1 + request: &geometry-request + num_tokens: 1800 + reference: + command: reference.py + precision: fp32 + gate: + mask_iou: 0.999 + depth_absrel_mean: 0.005 + depth_rel_l2: 0.02 + points_rel_l2: 0.02 + points_cosine: 0.99999 + intrinsics_max_relative_error: 0.002 + point_depth_consistency: 0.00001 + min_sample_pass_rate: 1.0 + +performance: + - name: geometry + benchmark: metric_geometry_performance + operation: geometry + request: + <<: *geometry-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: official_pytorch + precision: fp32 + output_contract: metric-geometry-parity + mask_iou: 0.999 + depth_absrel_mean: 0.005 + depth_rel_l2: 0.02 + points_rel_l2: 0.02 + points_cosine: 0.99999 + intrinsics_max_relative_error: 0.002 + point_depth_consistency: 0.00001 diff --git a/families/moge/tests/benchmark/prepare_environment.py b/families/moge/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4c525076e4 --- /dev/null +++ b/families/moge/tests/benchmark/prepare_environment.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned official MoGe source inside this family's environment.""" + +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +SOURCE_REPOSITORY = "https://github.com/microsoft/MoGe.git" +SOURCE_REVISION = "74fbce054ebed49800de42d0ad0e83495065719a" + + +def main() -> int: + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "MoGe" + required = destination / "moge/model/v2.py" + if required.is_file(): + return 0 + if destination.exists(): + raise RuntimeError(f"incomplete MoGe reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="MoGe.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", SOURCE_REPOSITORY], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(temporary), + "fetch", + "--quiet", + "--depth=1", + "origin", + SOURCE_REVISION, + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/moge/tests/benchmark/reference.py b/families/moge/tests/benchmark/reference.py new file mode 100644 index 0000000000..2e719b5180 --- /dev/null +++ b/families/moge/tests/benchmark/reference.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned official Accuracy and Performance reference for MoGe-2.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import statistics +import sys +import time +from typing import Any, Mapping + +import numpy as np + +from families.moge.tests.reference_support import OfficialReference, read_image + + +def _source_root() -> Path: + root = Path(sys.prefix) / "trtmc-reference/MoGe" + if not (root / "moge/model/v2.py").is_file(): + raise RuntimeError(f"MoGe reference is incomplete: {root}") + return root + + +def _checkpoint(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import hf_hub_download + + return Path( + hf_hub_download( + repo_id=model, + revision=revision, + filename="model.pt", + local_files_only=local_files_only, + ) + ) + + +def _options(request: Mapping[str, Any]) -> tuple[int, float | None]: + unknown = sorted(set(request) - {"image_path", "num_tokens", "fov_x"}) + if unknown: + raise ValueError("unsupported MoGe request fields: " + ", ".join(unknown)) + num_tokens = request.get("num_tokens") + if isinstance(num_tokens, bool) or not isinstance(num_tokens, int) or num_tokens != 1800: + raise ValueError("MoGe reference requires num_tokens=1800") + fov_x = request.get("fov_x") + if fov_x is not None: + fov_x = float(fov_x) + if not 0.0 < fov_x < 180.0: + raise ValueError("MoGe fov_x must be between zero and 180 degrees") + return num_tokens, fov_x + + +def _write_summary(arrays: Mapping[str, Any], prefix: Path) -> dict[str, Any]: + points = np.asarray(arrays["points"], dtype=" int: + payload = json.loads(arguments.request.read_text(encoding="utf-8")) + request = payload.get("request", {}) + samples = payload.get("samples") + if not isinstance(request, Mapping) or not isinstance(samples, list) or not samples: + raise ValueError("Accuracy reference requires request and samples") + num_tokens, fov_x = _options(request) + model = OfficialReference( + _source_root(), + _checkpoint( + str(payload["model"]), + payload.get("revision"), + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + ), + ) + output_samples = [] + for index, sample in enumerate(samples): + if not isinstance(sample, Mapping): + raise ValueError(f"Accuracy sample {index} must be an object") + arrays = model.infer( + read_image(Path(str(sample["image_path"]))), + num_tokens=num_tokens, + fov_x=fov_x, + ) + summary = _write_summary( + arrays, arguments.output.with_name(f"{arguments.output.stem}.{index}.geometry") + ) + output_samples.append({"sample_id": str(sample["sample_id"]), **summary}) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": output_samples}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +def _run_performance(arguments: argparse.Namespace) -> int: + import torch + + request = json.loads(arguments.request_json) + timing = json.loads(arguments.timing_contract_json) + num_tokens, fov_x = _options(request) + reference = OfficialReference( + _source_root(), + _checkpoint(arguments.model, arguments.revision, arguments.local_files_only), + ) + pixels = read_image(Path(str(request["image_path"]))) + + def invoke(): + return reference.infer(pixels, num_tokens=num_tokens, fov_x=fov_x) + + arrays = None + for _ in range(arguments.warmup): + arrays = invoke() + samples_ms = [] + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter_ns() + arrays = invoke() + torch.cuda.synchronize() + samples_ms.append((time.perf_counter_ns() - started) / 1_000_000) + if arrays is None: + raise RuntimeError("MoGe reference produced no geometry") + summary = _write_summary(arrays, arguments.output.with_suffix(".geometry")) + value = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + "framework": f"torch-{torch.__version__}", + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": timing, + **timing, + "samples_ms": samples_ms, + "metrics": {"latency_ms": {"p50": statistics.median(samples_ms)}}, + "output_summary": summary, + } + arguments.output.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + for name in ( + "model", + "family", + "manifest", + "operation", + "selected-task", + "request-json", + "adapter-options-json", + "timing-contract-json", + "precision", + "mode", + "padding", + "case-name", + ): + parser.add_argument("--" + name, required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--trust-remote-code", action="store_true") + return parser + + +def main() -> int: + if "--request" in sys.argv[1:]: + return _run_accuracy(_accuracy_parser().parse_args()) + return _run_performance(_performance_parser().parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/moge/tests/data/benchmark_geometry.json b/families/moge/tests/data/benchmark_geometry.json new file mode 100644 index 0000000000..67df60ae86 --- /dev/null +++ b/families/moge/tests/data/benchmark_geometry.json @@ -0,0 +1,8 @@ +{ + "requests": [ + { + "id": "moge-test-image", + "image": "test_img.jpeg" + } + ] +} diff --git a/families/moge/tests/reference_support.py b/families/moge/tests/reference_support.py new file mode 100644 index 0000000000..e3e2b04cc6 --- /dev/null +++ b/families/moge/tests/reference_support.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned pinned MoGe-2 reference loading and inference.""" + +from __future__ import annotations + +import contextlib +from pathlib import Path +import sys +import types + +import numpy as np + + +def _math_sdpa(torch): + try: + from torch.nn.attention import SDPBackend, sdpa_kernel + + return sdpa_kernel([SDPBackend.MATH]) + except (ImportError, AttributeError): + return contextlib.nullcontext() + + +def _install_utils3d_shim(torch) -> None: + def intrinsics_from_focal_center(fx, fy, cx, cy): + fx, fy, cx, cy = torch.broadcast_tensors( + torch.as_tensor(fx), + torch.as_tensor(fy), + torch.as_tensor(cx, device=torch.as_tensor(fx).device), + torch.as_tensor(cy, device=torch.as_tensor(fx).device), + ) + matrix = torch.zeros((*fx.shape, 3, 3), dtype=fx.dtype, device=fx.device) + matrix[..., 0, 0] = fx + matrix[..., 1, 1] = fy + matrix[..., 0, 2] = cx + matrix[..., 1, 2] = cy + matrix[..., 2, 2] = 1.0 + return matrix + + def depth_map_to_point_map(depth, *, intrinsics): + height, width = depth.shape[-2:] + u = (torch.arange(width, dtype=depth.dtype, device=depth.device) + 0.5) / width + v = (torch.arange(height, dtype=depth.dtype, device=depth.device) + 0.5) / height + u, v = torch.meshgrid(u, v, indexing="xy") + while u.ndim < depth.ndim: + u = u.unsqueeze(0) + v = v.unsqueeze(0) + x = (u - intrinsics[..., 0, 2, None, None]) / intrinsics[..., 0, 0, None, None] + y = (v - intrinsics[..., 1, 2, None, None]) / intrinsics[..., 1, 1, None, None] + return torch.stack((x * depth, y * depth, depth), dim=-1) + + shim = types.ModuleType("utils3d_moge") + shim.pt = types.SimpleNamespace( + intrinsics_from_focal_center=intrinsics_from_focal_center, + depth_map_to_point_map=depth_map_to_point_map, + ) + sys.modules["utils3d_moge"] = shim + sys.modules.setdefault("cv2", types.ModuleType("cv2")) + + +class OfficialReference: + def __init__(self, source_root: Path, checkpoint: Path): + import torch + + _install_utils3d_shim(torch) + source_root = source_root.resolve() + sys.path.insert(0, str(source_root)) + from moge.model.v2 import MoGeModel + + state = torch.load(checkpoint, map_location="cpu", weights_only=True, mmap=True) + if set(state) != {"model_config", "model"}: + raise ValueError("MoGe checkpoint has an unexpected top-level contract") + model = MoGeModel(**state["model_config"]) + missing, unexpected = model.load_state_dict(state["model"], strict=False) + if missing or unexpected: + raise ValueError(f"MoGe state mismatch: missing={missing}, unexpected={unexpected}") + model.onnx_compatible_mode = True + self.torch = torch + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model = model.eval().float().to(self.device) + if torch.cuda.is_available(): + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + def infer(self, pixels: np.ndarray, *, num_tokens: int, fov_x: float | None = None): + if num_tokens != 1800: + raise ValueError("the qualified MoGe reference requires num_tokens=1800") + tensor = self.torch.from_numpy(pixels).permute(2, 0, 1).to(self.device) + options = { + "num_tokens": num_tokens, + "use_fp16": False, + "force_projection": True, + "apply_mask": True, + } + if fov_x is not None: + options["fov_x"] = fov_x + with _math_sdpa(self.torch): + output = self.model.infer(tensor, **options) + required = {"points", "depth", "intrinsics", "mask"} + if set(output) != required: + raise ValueError( + f"MoGe reference returned {sorted(output)}, expected {sorted(required)}" + ) + return {name: tensor.detach().cpu().numpy() for name, tensor in output.items()} + + +def read_image(path: Path) -> np.ndarray: + from PIL import Image + + with Image.open(path) as image: + return np.asarray(image.convert("RGB"), dtype=np.float32).copy() / 255.0 diff --git a/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml b/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml new file mode 100644 index 0000000000..1c555ea9fd --- /dev/null +++ b/families/mpnet/tests/benchmark/all-mpnet-base-v2.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: all-mpnet-base-v2 +candidate: + family: mpnet + checkpoint: sentence-transformers/all-mpnet-base-v2 + revision: e8c3b32edf5434bc2275fc9bab85f82640a19130 + task: encoding + precision: fp16 + bundle: all-mpnet-base-v2.bundle + build: + max_sequence_length: 384 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml b/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml new file mode 100644 index 0000000000..1bf4939621 --- /dev/null +++ b/families/nemotron/tests/benchmark/nemotron-hindi-4b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-hindi-4b +candidate: + family: nemotron + checkpoint: nvidia/Nemotron-4-Mini-Hindi-4B-Base + revision: 05c6ef6d2ddaad06bce914990facaf08bc4c581e + task: text_generation + precision: bf16 + bundle: nemotron-hindi-4b.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: The capital of France is + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml b/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml new file mode 100644 index 0000000000..39de53d6ba --- /dev/null +++ b/families/nemotron/tests/benchmark/nemotron-mini-4b.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-mini-4b +candidate: + family: nemotron + checkpoint: nvidia/Nemotron-Mini-4B-Instruct + revision: 791833e92ebddb0bc2c1007f6d2b6764f886a2ae + task: text_generation + precision: fp16 + bundle: nemotron-mini-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml b/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml new file mode 100644 index 0000000000..0c163f4f01 --- /dev/null +++ b/families/nemotron_h/tests/benchmark/nemotron-h-nano-9b.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-h-nano-9b +candidate: + family: nemotron_h + checkpoint: nvidia/NVIDIA-Nemotron-Nano-9B-v2 + revision: 6533e8de2c68e4536bf7c411d7a3ce5734111476 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: nemotron-h-nano-9b.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + build_isolation: false + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: bf16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: bf16 + output_contract: exact-token-ids diff --git a/families/nemotron_h/tests/test_runtime_contract.py b/families/nemotron_h/tests/test_runtime_contract.py index e286648ca2..25a6d58e56 100644 --- a/families/nemotron_h/tests/test_runtime_contract.py +++ b/families/nemotron_h/tests/test_runtime_contract.py @@ -8,10 +8,19 @@ from types import SimpleNamespace import pytest +import yaml from families.nemotron_h.model import _runtime_config, _stop_token_ids +def test_qualification_environment_keeps_extension_builds_in_family_environment() -> None: + profile = yaml.safe_load( + (Path(__file__).parent / "benchmark/nemotron-h-nano-9b.yaml").read_text(encoding="utf-8") + ) + + assert profile["reference_environment"]["build_isolation"] is False + + def test_runtime_keeps_a_scalar_eos_and_every_declared_stop_token(tmp_path) -> None: (tmp_path / "generation_config.json").write_text( json.dumps({"eos_token_id": [2, 11, 12]}), encoding="utf-8" diff --git a/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml b/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml new file mode 100644 index 0000000000..886ffff711 --- /dev/null +++ b/families/nemotron_labs_diffusion/tests/benchmark/nemotron-labs-diffusion-8b.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-labs-diffusion-8b +candidate: + family: nemotron_labs_diffusion + checkpoint: nvidia/Nemotron-Labs-Diffusion-8B + revision: 16c67f0560b912e93e0cabb6e0c4f5c3086d95fc + task: text_generation + precision: bf16 + trust_remote_code: true + bundle: nemotron-labs-diffusion-8b.bundle + build: + max_sequence_length: 257 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: mmlu-ar-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + command: reference.py + precision: fp32 + request: &ar-generation + max_new_tokens: 32 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: true + enable_thinking: false + do_sample: false + text_generation_mode: ar + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-ar-8 + benchmark: text_generation_performance + operation: generate + request: + <<: *ar-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 8 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: hf-eager + model_class: auto + generation_method: ar-generate + precision: fp32 + output_contract: exact-token-ids diff --git a/families/nemotron_labs_diffusion/tests/benchmark/reference.py b/families/nemotron_labs_diffusion/tests/benchmark/reference.py new file mode 100644 index 0000000000..5d40a3ae4e --- /dev/null +++ b/families/nemotron_labs_diffusion/tests/benchmark/reference.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Nemotron Labs Diffusion autoregressive reference.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Sequence + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def _inputs(tokenizer: Any, prompt: str, generation: dict[str, Any]) -> Any: + if not generation.get("use_chat_template", False): + return tokenizer(prompt, return_tensors="pt") + rendered = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=bool(generation.get("enable_thinking", False)), + ) + return tokenizer(rendered, return_tensors="pt", add_special_tokens=False) + + +def _truncate(tokenizer: Any, prompt: str, limit: int, side: str) -> str: + if limit < 1 or side not in {"left", "right"}: + raise ValueError("prompt token limit and truncation side are invalid") + token_ids = [int(value) for value in tokenizer.encode(prompt, add_special_tokens=False)] + if len(token_ids) <= limit: + return prompt + selected = token_ids[-limit:] if side == "left" else token_ids[:limit] + return str( + tokenizer.decode( + selected, + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + ) + ) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + generation = request.get("generation") + if not isinstance(samples, list) or not samples or not isinstance(generation, dict): + raise ValueError("reference request must contain samples and generation") + mode = generation.get("text_generation_mode", generation.get("generation_mode", "ar")) + if mode != "ar": + raise ValueError("this qualification reference supports text_generation_mode=ar") + + import torch + from transformers import AutoModel, AutoTokenizer + + model_id = str(request["model"]) + revision = request.get("revision") + options = { + "trust_remote_code": bool(request.get("trust_remote_code", True)), + **({"revision": revision} if revision else {}), + } + tokenizer = AutoTokenizer.from_pretrained(model_id, **options) + model = AutoModel.from_pretrained( + model_id, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + **options, + ).eval().to("cuda") + max_new_tokens = int(generation.get("max_new_tokens", 20)) + prompt_token_limit = int(request.get("prompt_token_limit", 192)) + truncation_side = str(request.get("truncation_side", "left")) + results = [] + for sample in samples: + prompt = _truncate( + tokenizer, + str(sample["prompt"]), + prompt_token_limit, + truncation_side, + ) + encoded = _inputs(tokenizer, prompt, generation) + input_ids = encoded["input_ids"].to(model.device) + with torch.inference_mode(): + generated = model.ar_generate( + input_ids, + max_new_tokens=max_new_tokens, + temperature=float(generation.get("temperature", 0.0)), + eos_token_id=tokenizer.eos_token_id, + ) + if isinstance(generated, tuple): + generated = generated[0] + token_ids = generated[0, input_ids.shape[-1] :][:max_new_tokens].cpu().tolist() + results.append( + { + "sample_id": str(sample["sample_id"]), + "prompt": prompt, + "token_ids": token_ids, + "text": tokenizer.decode(token_ids, skip_special_tokens=True), + } + ) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/nemotron_labs_diffusion/tests/test_benchmark_reference.py b/families/nemotron_labs_diffusion/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..94a0861a34 --- /dev/null +++ b/families/nemotron_labs_diffusion/tests/test_benchmark_reference.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from families.nemotron_labs_diffusion.tests.benchmark.reference import _truncate + + +class _Tokenizer: + def encode(self, prompt: str, *, add_special_tokens: bool) -> list[int]: + assert add_special_tokens is False + return [int(value) for value in prompt.split()] + + def decode( + self, + token_ids: list[int], + *, + skip_special_tokens: bool, + clean_up_tokenization_spaces: bool, + ) -> str: + assert skip_special_tokens is False + assert clean_up_tokenization_spaces is False + return " ".join(str(value) for value in token_ids) + + +def test_benchmark_reference_applies_declared_prompt_window() -> None: + tokenizer = _Tokenizer() + + assert _truncate(tokenizer, "1 2 3 4", 2, "left") == "3 4" + assert _truncate(tokenizer, "1 2 3 4", 2, "right") == "1 2" diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml new file mode 100644 index 0000000000..ab37b766de --- /dev/null +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-3.5-asr-streaming-0.6b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-3.5-asr-streaming-0.6b +candidate: + family: nemotron_speech_streaming + checkpoint: nvidia/nemotron-3.5-asr-streaming-0.6b + revision: ea30d66debe3740a08b573244286791d423d6b3e + task: transcription_streaming + precision: fp16 + bundle: nemotron-3.5-asr-streaming-0.6b.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 80 + language: en-US + streaming: true + chunk_ms: 1120 + att_context_left: 56 + att_context_right: 13 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.35 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe-streaming + benchmark: speech_nemo_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: nemo_reference + precision: fp32 + output_contract: transcription-text diff --git a/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml new file mode 100644 index 0000000000..b7f8da14f8 --- /dev/null +++ b/families/nemotron_speech_streaming/tests/benchmark/nemotron-speech-streaming-en-0.6b.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: nemotron-speech-streaming-en-0.6b +candidate: + family: nemotron_speech_streaming + checkpoint: nvidia/nemotron-speech-streaming-en-0.6b + revision: ebe59e5a817142986528bbbee5dba8db7b38ed50 + task: transcription_streaming + precision: fp16 + bundle: nemotron-speech-streaming-en-0.6b.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 80 + streaming: true + chunk_ms: 1120 + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.35 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe-streaming + benchmark: speech_nemo_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: nemo_reference + precision: fp32 + output_contract: transcription-text diff --git a/families/nemotron_speech_streaming/tests/benchmark/reference.py b/families/nemotron_speech_streaming/tests/benchmark/reference.py new file mode 100644 index 0000000000..cfa90178ea --- /dev/null +++ b/families/nemotron_speech_streaming/tests/benchmark/reference.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned NeMo streaming-ASR reference for Accuracy qualification.""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import atexit +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.references import speech_io +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_audio_mono, + resample_linear, + transcription_text, + write_wav_pcm16, +) + + +def _text(value: Any) -> str: + return str(value.text if hasattr(value, "text") else value) + + +def _accuracy(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + import torch + from nemo.collections.asr.models import ASRModel + + model_id, revision = speech_io.model_id(request) + is_nemotron35 = "nemotron-3.5-asr-streaming" in model_id.casefold() + if is_nemotron35: + from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_nemotron35_asr_model, + ) + + model = load_nemotron35_asr_model( + model=model_id, + revision=revision or "", + local_files_only=False, + device="cuda", + ) + else: + model = ASRModel.from_pretrained(model_id, map_location="cpu").eval().to("cuda") + + language = request.get("language") + manifest = arguments.output.parent / "reference-audio" / "manifest.jsonl" + records = [] + for sample in samples: + value = { + "audio_filepath": str(sample.wav_path), + "duration": float(len(sample.waveform)) / sample.sample_rate, + "text": "", + } + if isinstance(language, str) and language: + value["lang"] = language + records.append(value) + manifest.write_text( + "".join(json.dumps(value, ensure_ascii=False) + "\n" for value in records), + encoding="utf-8", + ) + + original_forward = model.forward + if is_nemotron35: + + def forward_with_extended_prompt(*args: Any, **kwargs: Any) -> Any: + prompt = kwargs.get("prompt") + if prompt is not None and prompt.shape[1] > 0: + kwargs = dict(kwargs) + kwargs["prompt"] = torch.cat((prompt, prompt[:, -1:, :]), dim=1) + return original_forward(*args, **kwargs) + + model.forward = forward_with_extended_prompt + try: + options = {"batch_size": 1} + if is_nemotron35: + options["verbose"] = False + values = model.transcribe(str(manifest), **options) + finally: + model.forward = original_forward + if isinstance(values, tuple): + values = values[0] + speech_io.write_result(arguments.output, samples, [_text(value) for value in values]) + return 0 + + +def _disable_cuda_graphs(model: Any) -> None: + decoding = getattr(getattr(model, "cfg", None), "decoding", None) + if decoding is not None and hasattr(decoding, "use_cuda_graph_decoder"): + decoding.use_cuda_graph_decoder = False + change_strategy = getattr(model, "change_decoding_strategy", None) + if callable(change_strategy) and decoding is not None: + change_strategy(decoding_cfg=decoding) + + +def _performance_session( + arguments: Any, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager" or arguments.precision != "fp32": + raise ValueError("Nemotron streaming ASR reference requires pytorch-eager fp32") + import torch + from nemo.collections.asr.models import ASRModel + + is_nemotron35 = "nemotron-3.5-asr-streaming" in arguments.model.casefold() + if is_nemotron35: + from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + load_nemotron35_asr_model, + ) + + model = load_nemotron35_asr_model( + model=arguments.model, + revision=arguments.revision or "", + local_files_only=arguments.local_files_only, + device="cuda", + ) + else: + model = ASRModel.from_pretrained(arguments.model, map_location="cpu").eval().to("cuda") + _disable_cuda_graphs(model) + audio, source_rate = load_audio_mono(str(request["audio_path"])) + target_rate = 16_000 + audio = resample_linear(audio, source_rate, target_rate) + temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + temporary.close() + atexit.register(Path(temporary.name).unlink, missing_ok=True) + write_wav_pcm16(Path(temporary.name), audio, target_rate) + manifest = Path(temporary.name).with_suffix(".jsonl") + atexit.register(manifest.unlink, missing_ok=True) + record: dict[str, Any] = { + "audio_filepath": temporary.name, + "duration": float(len(audio)) / target_rate, + "text": "", + } + language = str(request.get("language", "") or "") + if language and language != "auto": + record["lang"] = language + manifest.write_text(json.dumps(record) + "\n", encoding="utf-8") + if is_nemotron35: + original_forward = model.forward + + def forward_with_extended_prompt(*args: Any, **kwargs: Any) -> Any: + prompt = kwargs.get("prompt") + if prompt is not None and prompt.shape[1] > 0: + kwargs = dict(kwargs) + kwargs["prompt"] = torch.cat((prompt, prompt[:, -1:, :]), dim=1) + return original_forward(*args, **kwargs) + + model.forward = forward_with_extended_prompt + + def invoke() -> Mapping[str, Any]: + options: dict[str, Any] = {"batch_size": 1} + if is_nemotron35: + options["verbose"] = False + values = model.transcribe(str(manifest), **options) + return {"text": transcription_text(values), "output_tokens": None} + + return reference_harness.Session( + invoke, + "nemo", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/olmo2/tests/benchmark/olmo2-1b.yaml b/families/olmo2/tests/benchmark/olmo2-1b.yaml deleted file mode 100644 index 8e7f23da7c..0000000000 --- a/families/olmo2/tests/benchmark/olmo2-1b.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: olmo2-1b -candidate: - family: olmo2 - checkpoint: allenai/OLMo-2-0425-1B - revision: a1847dff35000b4271fa70afc5db10fd29fedbdf - task: text_generation - precision: fp32 - bundle: olmo2-1b.bundle - build: - max_sequence_length: 352 - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp32 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-8 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: |- - The following are multiple choice questions (with answers) about abstract algebra. - - Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field. - A. 0 - B. 1 - C. 2 - D. 3 - Answer: B - - Statement 1 | If aH is an element of a factor group, then |aH| divides |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: B - - Statement 1 | Every element of a group generates a cyclic subgroup of the group. Statement 2 | The symmetric group S_10 has 10 elements. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: C - - Statement 1| Every function from a finite set onto itself must be one to one. Statement 2 | Every subgroup of an abelian group is abelian. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: A - - Find the characteristic of the ring 2Z. - A. 0 - B. 3 - C. 12 - D. 30 - Answer: A - - Find the degree for the given field extension Q(sqrt(2), sqrt(3), sqrt(18)) over Q. - A. 0 - B. 4 - C. 2 - D. 6 - Answer: - max_new_tokens: 8 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp32 - output_contract: exact-token-ids diff --git a/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml b/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml new file mode 100644 index 0000000000..488e7c5b15 --- /dev/null +++ b/families/patchtsmixer/tests/benchmark/patchtsmixer-granite-official.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: patchtsmixer-granite-official +candidate: + family: patchtsmixer + checkpoint: ibm-granite/granite-timeseries-patchtsmixer + revision: 90dc5a88d45f032b7dceefb5d814ca2af54f2ff9 + task: time_series_forecast + precision: fp16 + bundle: patchtsmixer-granite-official.bundle + build: + fp32_layers: [0, 1, 2, 3, 12] + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [HUFL, HULL, MUFL, MULL, LUFL, LULL, OT] + context_length: 512 + prediction_length: 96 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 5.0e-4 + max_absolute_error: 2.5e-2 + +performance: + - name: forecast-96 + benchmark: time_series_performance + operation: solve + request: + past_values: + [0.1, -0.2, 0.05, 0.3, -0.15, 0.25, 0.4, 0.2, -0.1, 0.1, + 0.35, -0.05, 0.2, 0.5, 0.3, 0.0, 0.15, 0.4, 0.05, 0.15, + 0.6, 0.4, 0.1, 0.2, 0.45, 0.1, 0.1, 0.7] + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + adapter_options: {reference_type: patchtsmixer} + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 5.0e-4 + max_absolute_error: 2.5e-2 diff --git a/families/patchtsmixer/tests/benchmark/reference.py b/families/patchtsmixer/tests/benchmark/reference.py new file mode 100644 index 0000000000..aa4655c019 --- /dev/null +++ b/families/patchtsmixer/tests/benchmark/reference.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""PatchTSMixer-owned ETTh1 Accuracy reference.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def prediction_model_class(transformers): + model_class = transformers.PatchTSMixerForPrediction + if not hasattr(model_class, "all_tied_weights_keys"): + model_class.all_tied_weights_keys = {} + return model_class + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if request.get("precision") != "fp32": + raise ValueError("PatchTSMixer Accuracy requires an fp32 reference") + if not torch.cuda.is_available(): + raise RuntimeError("PatchTSMixer Accuracy reference requires CUDA") + options = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + config = transformers.AutoConfig.from_pretrained(str(request["model"]), **options) + model_class = prediction_model_class(transformers) + model = ( + model_class.from_pretrained( + str(request["model"]), torch_dtype=torch.float32, **options + ) + .eval() + .to("cuda") + ) + context = int(config.context_length) + channels = int(config.num_input_channels) + samples = [] + with torch.inference_mode(): + for sample in request["samples"]: + raw = [float(value) for value in sample["past_values"]] + count = min(len(raw), context * channels) + values = [0.0] * (context * channels) + values[-count:] = raw[-count:] + observed = [0.0] * (context * channels) + observed[-count:] = [1.0] * count + output = model( + past_values=torch.tensor( + values, dtype=torch.float32, device="cuda" + ).reshape(1, context, channels), + observed_mask=torch.tensor( + observed, dtype=torch.float32, device="cuda" + ).reshape(1, context, channels), + return_loss=False, + return_dict=True, + ).prediction_outputs.detach().float().cpu() + samples.append( + { + "sample_id": str(sample["sample_id"]), + "shape": [int(size) for size in output.shape], + "values": output.reshape(-1).tolist(), + } + ) + arguments.output.write_text( + json.dumps( + { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": request.get("revision"), + "precision": "fp32", + "samples": samples, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/patchtsmixer/tests/test_benchmark_reference.py b/families/patchtsmixer/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..4e547c9cf0 --- /dev/null +++ b/families/patchtsmixer/tests/test_benchmark_reference.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from families.patchtsmixer.tests.benchmark.reference import prediction_model_class + + +def test_reference_adds_transformers_tied_weights_compatibility_field() -> None: + class PredictionModel: + pass + + assert not hasattr(PredictionModel, "all_tied_weights_keys") + + selected = prediction_model_class( + SimpleNamespace(PatchTSMixerForPrediction=PredictionModel) + ) + + assert selected is PredictionModel + assert selected.all_tied_weights_keys == {} diff --git a/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml b/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml new file mode 100644 index 0000000000..828864d1be --- /dev/null +++ b/families/patchtst/tests/benchmark/patchtst-etth1-regression-distribution.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: patchtst-etth1-regression-distribution +candidate: + family: patchtst + checkpoint: ibm-research/patchtst-etth1-regression-distribution + revision: d5ef731ed80ffcd2f0a3262138b1009bbcedb2c4 + task: time_series_forecast + precision: fp16 + bundle: patchtst-etth1-regression-distribution.bundle + build: + fp32_layers: [5] + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [HUFL, HULL, MUFL, MULL, LUFL, LULL] + context_length: 512 + prediction_length: 1 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 1.0e-3 + max_absolute_error: 1.0e-3 + +performance: + - name: regress + benchmark: time_series_performance + operation: solve + request: + past_values: + [0.1, -0.2, 0.05, 0.3, -0.15, 0.25, 0.4, 0.2, -0.1, 0.1, + 0.35, -0.05, 0.2, 0.5, 0.3, 0.0, 0.15, 0.4, 0.05, 0.15, + 0.6, 0.4, 0.1, 0.2, 0.45, 0.1, 0.1, 0.7, 0.55, 0.05] + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + adapter_options: {reference_type: patchtst} + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 1.0e-3 + max_absolute_error: 1.0e-3 diff --git a/families/patchtst/tests/benchmark/patchtst-granite-official.yaml b/families/patchtst/tests/benchmark/patchtst-granite-official.yaml new file mode 100644 index 0000000000..b75284dd56 --- /dev/null +++ b/families/patchtst/tests/benchmark/patchtst-granite-official.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: patchtst-granite-official +candidate: + family: patchtst + checkpoint: ibm-granite/granite-timeseries-patchtst + revision: 7fe295d8bc8fbac8041b60ab351882634165517f + task: time_series_forecast + precision: fp16 + bundle: patchtst-granite-official.bundle + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [HUFL, HULL, MUFL, MULL, LUFL, LULL, OT] + context_length: 512 + prediction_length: 96 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 1.5e-3 + max_absolute_error: 3.5e-2 + +performance: + - name: forecast-96 + benchmark: time_series_performance + operation: solve + request: + past_values: + [0.1, -0.2, 0.05, 0.3, -0.15, 0.25, 0.4, 0.2, -0.1, 0.1, + 0.35, -0.05, 0.2, 0.5, 0.3, 0.0, 0.15, 0.4, 0.05, 0.15, + 0.6, 0.4, 0.1, 0.2, 0.45, 0.1, 0.1, 0.7] + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + adapter_options: {reference_type: patchtst} + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 1.5e-3 + max_absolute_error: 3.5e-2 diff --git a/families/patchtst/tests/benchmark/reference.py b/families/patchtst/tests/benchmark/reference.py new file mode 100644 index 0000000000..ad2d36ff59 --- /dev/null +++ b/families/patchtst/tests/benchmark/reference.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""PatchTST-owned ETTh1 Accuracy reference.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if request.get("precision") != "fp32": + raise ValueError("PatchTST Accuracy requires an fp32 reference") + if not torch.cuda.is_available(): + raise RuntimeError("PatchTST Accuracy reference requires CUDA") + options = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + config = transformers.AutoConfig.from_pretrained(str(request["model"]), **options) + architecture = " ".join(getattr(config, "architectures", ()) or ()).lower() + if "regression" in architecture: + model_class = transformers.PatchTSTForRegression + output_name = "regression_outputs" + else: + model_class = transformers.PatchTSTForPrediction + output_name = "prediction_outputs" + model = ( + model_class.from_pretrained( + str(request["model"]), torch_dtype=torch.float32, **options + ) + .eval() + .to("cuda") + ) + context = int(config.context_length) + channels = int(config.num_input_channels) + samples = [] + with torch.inference_mode(): + for sample in request["samples"]: + raw = [float(value) for value in sample["past_values"]] + count = min(len(raw), context * channels) + values = [0.0] * (context * channels) + values[-count:] = raw[-count:] + observed = [False] * (context * channels) + observed[-count:] = [True] * count + outputs = model( + past_values=torch.tensor( + values, dtype=torch.float32, device="cuda" + ).reshape(1, context, channels), + past_observed_mask=torch.tensor( + observed, dtype=torch.bool, device="cuda" + ).reshape(1, context, channels), + return_dict=True, + ) + output = getattr(outputs, output_name) + if isinstance(output, tuple): + output = torch.stack(output, dim=-1) + output = output.detach().float().cpu() + samples.append( + { + "sample_id": str(sample["sample_id"]), + "shape": [int(size) for size in output.shape], + "values": output.reshape(-1).tolist(), + } + ) + arguments.output.write_text( + json.dumps( + { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": request.get("revision"), + "precision": "fp32", + "samples": samples, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/personaplex/tests/benchmark/personaplex-7b.yaml b/families/personaplex/tests/benchmark/personaplex-7b.yaml new file mode 100644 index 0000000000..23b5ced569 --- /dev/null +++ b/families/personaplex/tests/benchmark/personaplex-7b.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: personaplex-7b +candidate: {family: personaplex, checkpoint: nvidia/personaplex-7b-v1, revision: fdaf4090a61cb315c138a1faee287ffd6c716309, task: speech_to_speech, precision: fp16, bundle: personaplex-7b.bundle, build: {max_sequence_length: 512, fp32_layers: [0, 1]}} +reference_environment: + requirements: ../../requirements.txt + paths: {official_repo: trtmc-reference/personaplex} +accuracy: + - name: generated-audio-parity + benchmark: task_output_parity + operation: speak + request: &request {audio_path: ../data/Recording.wav, max_new_tokens: 300} + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: bf16, output_contract: audio-artifact-parity} + gate: {min_duration_ratio: 0.8, max_duration_ratio: 1.2, min_rms_ratio: 0.5, max_rms_ratio: 2.0, max_log_spectral_distance: 3.0} +performance: + - {name: speak, benchmark: audio_generation_pipeline_performance, operation: speak, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: torch_reference, output_contract: audio-shape}} diff --git a/families/personaplex/tests/benchmark/prepare_environment.py b/families/personaplex/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..9f7b9140d0 --- /dev/null +++ b/families/personaplex/tests/benchmark/prepare_environment.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare the pinned family-owned PersonaPlex reference checkout.""" + +from importlib.metadata import PackageNotFoundError, version as package_version +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +REPOSITORY = "https://github.com/NVIDIA/personaplex.git" +REVISION = "3428dfd95309a7f3c84fd93259ded0f810d1ff91" +SPHN_VERSION = "0.1.4" + + +def _install_sphn() -> None: + try: + installed = package_version("sphn") + except PackageNotFoundError: + installed = None + if installed == SPHN_VERSION: + return + environment = dict(os.environ) + # sphn vendors an older Opus CMake project. CMake 4 requires its legacy + # policy floor to be selected explicitly when building on aarch64. + environment.setdefault("CMAKE_POLICY_VERSION_MINIMUM", "3.5") + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + f"sphn=={SPHN_VERSION}", + ], + check=True, + env=environment, + ) + + +def main() -> None: + _install_sphn() + parent = Path(sys.prefix) / "trtmc-reference" + destination = parent / "personaplex" + if (destination / "moshi").is_dir(): + return + if destination.exists(): + raise RuntimeError(f"incomplete PersonaPlex reference: {destination}") + parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix="personaplex.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", REPOSITORY], check=True + ) + subprocess.run( + ["git", "-C", str(temporary), "fetch", "--quiet", "--depth=1", "origin", REVISION], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +if __name__ == "__main__": + main() diff --git a/families/personaplex/tests/benchmark/reference.py b/families/personaplex/tests/benchmark/reference.py new file mode 100644 index 0000000000..ef0cb1b69d --- /dev/null +++ b/families/personaplex/tests/benchmark/reference.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned PersonaPlex reference.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _materialize(summary: dict[str, Any], output: Path) -> None: + import soundfile as sound + + audio = summary.pop("_audio_f32") + artifact = output.with_suffix(".audio.wav").resolve() + artifact.parent.mkdir(parents=True, exist_ok=True) + sound.write(artifact, audio, int(summary["sample_rate"]), subtype="FLOAT") + summary["audio_artifact"] = str(artifact) + + +def _load( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "pytorch-eager": + raise ValueError("PersonaPlex reference requires pytorch-eager mode") + official_repo = Path(str(options.get("official_repo", ""))).resolve() + if not official_repo.is_dir(): + raise ValueError("PersonaPlex reference requires official_repo") + if importlib.util.find_spec("sphn") is None: + raise RuntimeError("PersonaPlex reference requires the official sphn package") + sys.path[:0] = [str(official_repo / "moshi"), str(official_repo)] + import torch + from huggingface_hub import hf_hub_download + from moshi.models import LMGen, loaders + from moshi.models.lm import _iterate_audio, encode_from_sphn, load_audio + from moshi.offline import warmup + + device = "cuda" + common = { + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + common["revision"] = arguments.revision + mimi_weights = hf_hub_download(arguments.model, loaders.MIMI_NAME, **common) + model_weights = hf_hub_download(arguments.model, loaders.MOSHI_NAME, **common) + mimi = loaders.get_mimi(mimi_weights, device) + other_mimi = loaders.get_mimi(mimi_weights, device) + language_model = loaders.get_moshi_lm(model_weights, device=device).eval() + frame_size = int(mimi.sample_rate / mimi.frame_rate) + generator = LMGen( + language_model, + audio_silence_frame_cnt=0, + sample_rate=mimi.sample_rate, + device=device, + frame_rate=mimi.frame_rate, + use_sampling=False, + temp=0.8, + temp_text=0.7, + top_k=250, + top_k_text=25, + ) + mimi.streaming_forever(1) + other_mimi.streaming_forever(1) + generator.streaming_forever(1) + warmup(mimi, other_mimi, generator, device, frame_size) + source_audio = load_audio(str(request["audio_path"]), mimi.sample_rate) + max_frames = int(request.get("max_new_tokens", options.get("max_frames", 100))) + + def invoke() -> Mapping[str, Any]: + mimi.reset_streaming() + other_mimi.reset_streaming() + generator.reset_streaming() + generated_frames = 0 + decoded_chunks = [] + + def summary() -> Mapping[str, Any]: + audio = ( + torch.cat(decoded_chunks, dim=-1).detach().float().cpu().reshape(-1).numpy() + if decoded_chunks + else torch.empty(0, dtype=torch.float32).numpy() + ) + return { + "audio_frames": generated_frames, + "audio_samples": int(audio.size), + "sample_rate": mimi.sample_rate, + "_audio_f32": audio, + } + + with torch.inference_mode(): + for encoded in encode_from_sphn( + mimi, + _iterate_audio(source_audio, sample_interval_size=frame_size, pad=True), + max_batch=1, + ): + for index in range(encoded.shape[-1]): + tokens = generator.step(encoded[:, :, index : index + 1]) + if tokens is None: + continue + decoded_chunks.append(mimi.decode(tokens[:, 1:9])) + other_mimi.decode(tokens[:, 1:9]) + generated_frames += 1 + if generated_frames >= max_frames: + return summary() + return summary() + + return reference_harness.Session( + invoke, + "moshi", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + materialize=_materialize, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run( + list(sys.argv[1:] if argv is None else argv), description=__doc__, load=_load + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/personaplex/tests/test_official_reference.py b/families/personaplex/tests/test_official_reference.py index b14c2fcf36..d29af04f46 100644 --- a/families/personaplex/tests/test_official_reference.py +++ b/families/personaplex/tests/test_official_reference.py @@ -14,6 +14,7 @@ import pytest from . import official_reference +from .benchmark import prepare_environment from .personaplex_audio_compat import sphn @@ -21,6 +22,23 @@ SOURCE_REVISION = "3428dfd95309a7f3c84fd93259ded0f810d1ff91" +def test_qualification_environment_installs_official_audio_dependency(monkeypatch) -> None: + calls = [] + monkeypatch.setattr(prepare_environment, "package_version", lambda _name: "0.1.3") + monkeypatch.setattr( + prepare_environment.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)), + ) + + prepare_environment._install_sphn() + + command, kwargs = calls[0] + assert command[-1] == "sphn==0.1.4" + assert kwargs["check"] is True + assert kwargs["env"]["CMAKE_POLICY_VERSION_MINIMUM"] == "3.5" + + def _write_wav(path: Path) -> None: with wave.open(str(path), "wb") as output: output.setnchannels(1) diff --git a/families/phi/tests/benchmark/phi3-mini.yaml b/families/phi/tests/benchmark/phi3-mini.yaml new file mode 100644 index 0000000000..a46f7a20ad --- /dev/null +++ b/families/phi/tests/benchmark/phi3-mini.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: phi3-mini +candidate: + family: phi + checkpoint: microsoft/Phi-3-mini-4k-instruct + revision: f39ac1d28e925b323eae81227eaba4464caced4e + task: text_generation + precision: fp16 + bundle: phi3-mini.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the boiling point of water in Celsius? Answer with just the number. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml b/families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml new file mode 100644 index 0000000000..d808863a1e --- /dev/null +++ b/families/phi4_multimodal/tests/benchmark/phi4-multimodal.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: phi4-multimodal +candidate: + family: phi4_multimodal + checkpoint: microsoft/Phi-4-multimodal-instruct + revision: 93f923e1a7727d1c4f446756212d9d3e8fcc5d81 + task: vision_language_generation + precision: fp16 + bundle: phi4-multimodal.bundle + build: + max_sequence_length: 768 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/phi4_multimodal/tests/benchmark/reference.py b/families/phi4_multimodal/tests/benchmark/reference.py new file mode 100644 index 0000000000..fa4d47d0c0 --- /dev/null +++ b/families/phi4_multimodal/tests/benchmark/reference.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers Phi-4 Multimodal reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor + + try: + dtype = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True) + model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True) + model_config._attn_implementation = "eager" + model_config._attn_implementation_internal = "eager" + model = ( + AutoModelForCausalLM.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=dtype, + config=model_config, + attn_implementation="eager", + ) + .eval() + .to("cuda") + ) + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + rendered = reference.processor.tokenizer.apply_chat_template( + [{"role": "user", "content": f"<|image_1|>{prompt}"}], + tokenize=False, + add_generation_prompt=True, + ) + if ( + not isinstance(rendered, str) + or not rendered.strip() + or not any( + marker in rendered + for marker in ("<|image_1|>", "<|image_pad|>", "<|vision_start|>", "") + ) + ): + raise ValueError("Phi-4 Multimodal chat template lost the image placeholder") + encoded = reference.processor(text=rendered, images=image, return_tensors="pt") + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if hasattr(value, "to") + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any], max_new_tokens: int) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + generated = reference.model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = generated[0] + tokens = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in tokens.detach().cpu().tolist()] + return { + "text": reference.processor.decode(tokens, skip_special_tokens=True), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + prompt = str(controls.get("prompt", "")) + max_new_tokens = int(controls.get("max_new_tokens", 10)) + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append( + { + "sample_id": str(sample["sample_id"]), + **_infer(reference, inputs, max_new_tokens), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "phi4_multimodal" or arguments.operation != "generate": + raise ValueError("Phi-4 Multimodal reference only supports phi4_multimodal generate") + if arguments.mode != "pytorch-eager": + raise ValueError("Phi-4 Multimodal reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request.get("prompt", ""))) + max_new_tokens = int(request.get("max_new_tokens", 10)) + for _ in range(arguments.warmup): + _infer(reference, inputs, max_new_tokens) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs, max_new_tokens) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/phi_moe/tests/benchmark/phi-moe.yaml b/families/phi_moe/tests/benchmark/phi-moe.yaml new file mode 100644 index 0000000000..cd4fe380ab --- /dev/null +++ b/families/phi_moe/tests/benchmark/phi-moe.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: phi-moe +candidate: + family: phi_moe + checkpoint: microsoft/Phi-tiny-MoE-instruct + revision: 2fe50e88d0e2a5a132563815686ea0dcc8e252b5 + task: text_generation + precision: fp16 + bundle: phi-moe.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: Name the largest ocean on Earth. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + experts_implementation: batched_mm + precision: fp32 + output_contract: exact-token-ids diff --git a/families/pixart/tests/benchmark/pixart-sigma-1024.yaml b/families/pixart/tests/benchmark/pixart-sigma-1024.yaml new file mode 100644 index 0000000000..c20fc721f1 --- /dev/null +++ b/families/pixart/tests/benchmark/pixart-sigma-1024.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: pixart-sigma-1024 +candidate: + family: pixart + checkpoint: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS + revision: e102b3591cc82e97071b8b4cb90d834d0c487207 + task: image_generation + precision: fp16 + bundle: pixart-sigma-1024.bundle + build: {fp32_layers: [0], image_height: 1024, image_width: 1024} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 20, height: 1024, width: 1024} + reference: &reference + adapter: hf-diffusers + mode: hf-eager + precision: fp16 + output_contract: media-artifact-parity + adapter_options: {component_precision_contract: pixart_fp16_dit_fp32_t5} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - name: generate-image + benchmark: media_generation_performance + operation: generate_image + request: *request + measurement: {warmup: 5, iterations: 10} + reference: + <<: *reference + runner: task-reference + reference_backend: hf_diffusers + output_contract: media-shape diff --git a/families/opt/tests/benchmark/opt-125m.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml similarity index 80% rename from families/opt/tests/benchmark/opt-125m.yaml rename to families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml index fa343f9ec3..e3aa7ca591 100644 --- a/families/opt/tests/benchmark/opt-125m.yaml +++ b/families/qwen/tests/benchmark/qwen3-0.6b-fp16.yaml @@ -2,15 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: opt-125m +model: qwen3-0.6b-fp16 candidate: - family: opt - checkpoint: facebook/opt-125m + family: qwen + checkpoint: Qwen/Qwen3-0.6B + revision: c1899de289a04d12100db370d81485cdf75e47ca task: text_generation precision: fp16 - bundle: opt-125m.bundle + bundle: qwen3-0.6b-fp16.bundle build: - max_sequence_length: 256 + max_sequence_length: 257 accuracy: - name: mmlu-continuation @@ -41,8 +42,9 @@ performance: operation: generate request: <<: *greedy-generation - prompt: The capital of France is + prompt: What is the capital of France? Answer in one word. max_new_tokens: 10 + use_chat_template: true measurement: warmup: 5 iterations: 10 diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml new file mode 100644 index 0000000000..4f8348834e --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-0.6b-fp8.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-0.6b-fp8 +candidate: + family: qwen + checkpoint: Qwen/Qwen3-0.6B + revision: c1899de289a04d12100db370d81485cdf75e47ca + task: text_generation + precision: fp16 + bundle: qwen3-0.6b-fp8-fp16base.bundle + build: + max_sequence_length: 257 + quantization: fp8 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp16 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-1 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: "What place is named in the title of the 1979 live album by rock legends Cheap Trick? Answer:" + max_new_tokens: 1 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp16 + output_contract: exact-token-ids diff --git a/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml b/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml new file mode 100644 index 0000000000..e0d35d76d9 --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-0.6b-topp.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-0.6b-topp +candidate: + family: qwen + checkpoint: Qwen/Qwen3-0.6B + revision: c1899de289a04d12100db370d81485cdf75e47ca + task: text_generation + precision: fp16 + bundle: qwen3-0.6b-topp-fp16.bundle + build: {max_sequence_length: 257} +accuracy: + - name: mmlu-sampling + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: {precision: fp32} + request: &sampling + {max_new_tokens: 20, temperature: 0.7, top_k: 50, top_p: 0.9, min_p: 0.0, seed: 42, repetition_penalty: 1.0, use_chat_template: false, enable_thinking: false, do_sample: true} + gate: {min_pass_rate: 0.9, allowed_failures: 1} +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *sampling + prompt: What is the capital of France? Answer in one word. + use_chat_template: true + measurement: {warmup: 5, iterations: 10} + reference: {runner: hf-transformers, mode: torch-compile, fallback: hf-eager, compile_scope: model.forward, precision: fp32, output_contract: exact-token-ids} diff --git a/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml b/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml new file mode 100644 index 0000000000..502b2f4bb6 --- /dev/null +++ b/families/qwen/tests/benchmark/qwen3-4b-instruct-2507.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-4b-instruct-2507 +candidate: + family: qwen + checkpoint: Qwen/Qwen3-4B-Instruct-2507 + revision: cdbee75f17c01a7cc42f958dc650907174af0554 + task: text_generation + precision: fp16 + bundle: qwen3-4b-instruct-2507.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of Japan? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml b/families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml new file mode 100644 index 0000000000..081c2ace45 --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-0.8b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-0.8b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-0.8B + revision: 2fc06364715b967f1860aea9cf38778875588b17 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-0.8b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-2b.yaml b/families/qwen3_5/tests/benchmark/qwen35-2b.yaml new file mode 100644 index 0000000000..c73616e6ce --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-2b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-2b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-2B + revision: 15852e8c16360a2fea060d615a32b45270f8a8fc + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-2b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-4b.yaml b/families/qwen3_5/tests/benchmark/qwen35-4b.yaml new file mode 100644 index 0000000000..f9ffc656cc --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-4b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-4b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-4B + revision: 851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-4b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_5/tests/benchmark/qwen35-9b.yaml b/families/qwen3_5/tests/benchmark/qwen35-9b.yaml new file mode 100644 index 0000000000..e6d3a8e00d --- /dev/null +++ b/families/qwen3_5/tests/benchmark/qwen35-9b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen35-9b +candidate: + family: qwen3_5 + checkpoint: Qwen/Qwen3.5-9B + revision: c202236235762e1c871ad0ccb60c8ee5ba337b9a + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen35-9b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_8/tests/benchmark/qwen36-27b.yaml b/families/qwen3_8/tests/benchmark/qwen36-27b.yaml new file mode 100644 index 0000000000..7c41261e14 --- /dev/null +++ b/families/qwen3_8/tests/benchmark/qwen36-27b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen36-27b +candidate: + family: qwen3_8 + checkpoint: Qwen/Qwen3.6-27B + revision: 6a9e13bd6fc8f0983b9b99948120bc37f49c13e9 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen36-27b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_8/tests/benchmark/qwen38-27b.yaml b/families/qwen3_8/tests/benchmark/qwen38-27b.yaml new file mode 100644 index 0000000000..1c6aa382af --- /dev/null +++ b/families/qwen3_8/tests/benchmark/qwen38-27b.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen38-27b +candidate: + family: qwen3_8 + checkpoint: Qwen/Qwen3.8-27B + revision: 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 + task: text_generation + precision: fp16 + trust_remote_code: true + bundle: qwen38-27b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the capital of France? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml b/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml new file mode 100644 index 0000000000..a07bb8bd07 --- /dev/null +++ b/families/qwen3_omni/tests/benchmark/qwen3-omni-30b-a3b-instruct.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen3-omni-30b-a3b-instruct +candidate: {family: qwen3_omni, checkpoint: Qwen/Qwen3-Omni-30B-A3B-Instruct, revision: 26291f793822fb6be9555850f06dfe95f2d7e695, task: text_generation, precision: bf16, bundle: qwen3-omni-30b-a3b-instruct.bundle, build: {max_sequence_length: 256}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-text-parity + benchmark: task_output_parity + operation: generate + request: &request {prompt: Please say hello from Qwen3 Omni in one short sentence., max_new_tokens: 16} + reference: &reference {script: tests/benchmark/reference.py, mode: hf-eager, precision: bf16, output_contract: normalized-text} + gate: {max_normalized_edit_distance: 0.15} +performance: + - {name: generate, benchmark: vision_language_performance, operation: generate, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_transformers, output_contract: exact-text}} diff --git a/families/qwen3_omni/tests/benchmark/reference.py b/families/qwen3_omni/tests/benchmark/reference.py new file mode 100644 index 0000000000..e7ad4c51d9 --- /dev/null +++ b/families/qwen3_omni/tests/benchmark/reference.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned Qwen3-Omni text reference.""" + +from __future__ import annotations + +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +SYSTEM_PROMPT = ( + "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " + "capable of perceiving auditory and visual inputs, as well as generating " + "text and speech." +) +CHAT_TEMPLATE = """{%- for message in messages %} +{{- '<|im_start|>' + message.role + '\n' }} +{%- if message.content is string %} +{{- message.content }} +{%- else %} +{%- for item in message.content %} +{%- if item.type == 'text' %}{{- item.text }}{%- endif %} +{%- endfor %} +{%- endif %} +{{- '<|im_end|>\n' }} +{%- endfor %} +{%- if add_generation_prompt %}{{- '<|im_start|>assistant\n' }}{%- endif %}""" + + +def _load( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Session: + if arguments.mode != "hf-eager": + raise ValueError("Qwen3-Omni reference requires hf-eager mode") + import torch + from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor + + common = { + "trust_remote_code": arguments.trust_remote_code, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + common["revision"] = arguments.revision + processor = Qwen3OmniMoeProcessor.from_pretrained(arguments.model, **common) + dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[ + arguments.precision + ] + model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( + arguments.model, + torch_dtype=dtype, + device_map=str(options.get("device_map", "cuda:0")), + enable_audio_output=False, + **common, + ).eval() + conversation = [ + {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT}]}, + {"role": "user", "content": [{"type": "text", "text": str(request.get("prompt", ""))}]}, + ] + processor_template = getattr(processor, "chat_template", None) + tokenizer_template = getattr(getattr(processor, "tokenizer", None), "chat_template", None) + template = ( + processor_template + if isinstance(processor_template, str) and processor_template.strip() + else tokenizer_template + if isinstance(tokenizer_template, str) and tokenizer_template.strip() + else CHAT_TEMPLATE + ) + inputs = processor.apply_chat_template( + conversation, + chat_template=template, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding=True, + ).to(model.device) + max_new_tokens = int(request.get("max_new_tokens", 16)) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + text_ids = model.generate( + **inputs, + thinker_max_new_tokens=max_new_tokens, + thinker_do_sample=False, + return_audio=False, + ) + generated = text_ids[:, int(inputs["input_ids"].shape[-1]) :] + token_ids = [int(token) for token in generated[0].detach().cpu().tolist()] + return { + "text": processor.batch_decode(generated, skip_special_tokens=True)[0].strip(), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run( + list(sys.argv[1:] if argv is None else argv), description=__doc__, load=_load + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/qwen3_omni/tests/test_benchmark_reference.py b/families/qwen3_omni/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..664a81a9bd --- /dev/null +++ b/families/qwen3_omni/tests/test_benchmark_reference.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def test_text_performance_reference_is_family_owned() -> None: + profile = yaml.safe_load( + (ROOT / "benchmark/qwen3-omni-30b-a3b-instruct.yaml").read_text(encoding="utf-8") + ) + case = profile["performance"][0] + assert case["operation"] == "generate" + assert case["request"]["max_new_tokens"] == 16 + assert case["reference"]["script"] == "tests/benchmark/reference.py" + assert "adapter" not in case["reference"] + + source = (ROOT / "benchmark/reference.py").read_text(encoding="utf-8") + assert "enable_audio_output=False" in source + assert "return_audio=False" in source diff --git a/families/qwen_image/tests/benchmark/qwen-image-2512.yaml b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml new file mode 100644 index 0000000000..6bf60f4e27 --- /dev/null +++ b/families/qwen_image/tests/benchmark/qwen-image-2512.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen-image-2512 +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-2512, revision: 25468b98e3276ca6700de15c6628e51b7de54a26, task: image_generation, precision: bf16, bundle: qwen-image-2512.bundle, build: {image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - {name: generated-media-parity, benchmark: task_output_parity, operation: generate_image, request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024}, reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true, always_negative_prompt: true, guidance_parameter: true_cfg_scale}}, gate: {min_psnr: 5.0, min_ssim: 0.1}} +performance: + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 10, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml b/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml new file mode 100644 index 0000000000..f9b14d7a25 --- /dev/null +++ b/families/qwen_image/tests/benchmark/qwen-image-edit-2511.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen-image-edit-2511 +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image-Edit-2511, revision: 6f3ccc0b56e431dc6a0c2b2039706d7d26f22cb9, task: image_edit, precision: bf16, bundle: qwen-image-edit-2511.bundle, build: {image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: Turn the image into a watercolor illustration while preserving the main subject., image_path: ../data/test_img.jpeg, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true, always_negative_prompt: true, guidance_parameter: true_cfg_scale}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: edit-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_image/tests/benchmark/qwen-image.yaml b/families/qwen_image/tests/benchmark/qwen-image.yaml new file mode 100644 index 0000000000..9103502bfc --- /dev/null +++ b/families/qwen_image/tests/benchmark/qwen-image.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: qwen-image +candidate: {family: qwen_image, checkpoint: Qwen/Qwen-Image, revision: 75e0b4be04f60ec59a75f475837eced720f823b6, task: image_generation, precision: bf16, bundle: qwen-image.bundle, build: {image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A cat holding a sign that says hello world, negative_prompt: ' ', seed: 42, num_steps: 20, cfg_scale: 4.0, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {cpu_offload: true, always_negative_prompt: true, guidance_parameter: true_cfg_scale}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml b/families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml new file mode 100644 index 0000000000..2ac4e54460 --- /dev/null +++ b/families/qwen_moe/tests/benchmark/qwen3-moe-30b-a3b.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-moe-30b-a3b +candidate: + family: qwen_moe + checkpoint: Qwen/Qwen3-30B-A3B + revision: ad44e777bcd18fa416d9da3bd8f70d33ebb85d39 + task: text_generation + precision: fp16 + bundle: qwen3-moe-30b-a3b.bundle + build: + max_sequence_length: 129 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 64 + truncation_side: left + reference: + precision: fp32 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the largest planet in our solar system? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + experts_implementation: batched_mm + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml b/families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml new file mode 100644 index 0000000000..b3d6da5846 --- /dev/null +++ b/families/qwen_moe/tests/benchmark/qwen3-moe-tiny-random.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-moe-tiny-random +candidate: + family: qwen_moe + checkpoint: amd-quark/tiny-random-qwen3_moe + revision: ca2aaa9a82e9f86c09c2785b405932fe6c806c90 + task: text_generation + precision: fp16 + bundle: qwen3-moe-tiny-random.bundle + build: + max_sequence_length: 129 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 64 + truncation_side: left + reference: + precision: fp32 + experts_implementation: batched_mm + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-10 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: What is the largest planet in our solar system? Answer in one word. + max_new_tokens: 10 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + experts_implementation: batched_mm + precision: fp32 + output_contract: exact-token-ids diff --git a/families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml b/families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml new file mode 100644 index 0000000000..b5c0c52694 --- /dev/null +++ b/families/qwen_vl/tests/benchmark/qwen25vl-3b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen25vl-3b +candidate: + family: qwen_vl + checkpoint: Qwen/Qwen2.5-VL-3B-Instruct + revision: 66285546d2b821cf421d4f5eb2576359d3770cd3 + task: vision_language_generation + precision: fp32 + bundle: qwen25vl-3b-vl.bundle + build: + max_sequence_length: 384 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: fp32 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml b/families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml new file mode 100644 index 0000000000..eb637c6190 --- /dev/null +++ b/families/qwen_vl/tests/benchmark/qwen3-vl-2b.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: qwen3-vl-2b +candidate: + family: qwen_vl + checkpoint: Qwen/Qwen3-VL-2B-Instruct + revision: 89644892e4d85e24eaac8bacfd4f463576704203 + task: vision_language_generation + precision: bf16 + bundle: qwen3-vl-2b.bundle + build: + max_sequence_length: 256 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-vlm-parity + benchmark: imagenette_vision_language_parity + samples: 10 + request: &vlm-request + prompt: What is the main object in this image? Answer in one word. + max_new_tokens: 10 + temperature: 1.0 + top_k: 1 + reference: + command: reference.py + precision: bf16 + gate: + max_normalized_edit_distance: 0.15 + min_sample_pass_rate: 0.9 + +performance: + - name: generate + benchmark: vision_language_performance + operation: generate + request: + <<: *vlm-request + image_path: ../data/test_img.jpeg + prompt: What color is the vehicle in this image? Answer in one word. + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: bf16 + output_contract: vision-language-text + max_normalized_edit_distance: 0.15 diff --git a/families/qwen_vl/tests/benchmark/reference.py b/families/qwen_vl/tests/benchmark/reference.py new file mode 100644 index 0000000000..b6a7ba3c72 --- /dev/null +++ b/families/qwen_vl/tests/benchmark/reference.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers Qwen-VL reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import AutoModelForImageTextToText, AutoProcessor + + try: + dtype = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, + }[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True) + model = ( + AutoModelForImageTextToText.from_pretrained( + model_dir, + trust_remote_code=True, + torch_dtype=dtype, + ) + .eval() + .to("cuda") + ) + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + rendered = reference.processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image", "image": image_path}, + {"type": "text", "text": prompt}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + if not isinstance(rendered, str) or not rendered.strip(): + raise ValueError("Qwen-VL chat template produced an empty prompt") + if not any(marker in rendered for marker in ("<|image_pad|>", "<|vision_start|>", "")): + raise ValueError("Qwen-VL chat template lost the image placeholder") + encoded = reference.processor(text=rendered, images=image, padding=True, return_tensors="pt") + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if hasattr(value, "to") + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any], max_new_tokens: int) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + generated = reference.model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = generated[0] + tokens = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in tokens.detach().cpu().tolist()] + return { + "text": reference.processor.decode(tokens, skip_special_tokens=True), + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + prompt = str(controls.get("prompt", "")) + max_new_tokens = int(controls.get("max_new_tokens", 10)) + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append( + { + "sample_id": str(sample["sample_id"]), + **_infer(reference, inputs, max_new_tokens), + } + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "qwen_vl" or arguments.operation != "generate": + raise ValueError("Qwen-VL reference only supports qwen_vl generate") + if arguments.mode != "pytorch-eager": + raise ValueError("Qwen-VL reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request.get("prompt", ""))) + max_new_tokens = int(request.get("max_new_tokens", 10)) + for _ in range(arguments.warmup): + _infer(reference, inputs, max_new_tokens) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs, max_new_tokens) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/roberta/tests/benchmark/camembert-base.yaml b/families/roberta/tests/benchmark/camembert-base.yaml new file mode 100644 index 0000000000..95f63182ac --- /dev/null +++ b/families/roberta/tests/benchmark/camembert-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: camembert-base +candidate: + family: roberta + checkpoint: almanach/camembert-base + revision: a75967561c78f2aa81cc41045378d3b4ee25af9e + task: encoding + precision: fp16 + bundle: camembert-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: Bonjour le monde. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/roberta/tests/benchmark/roberta-base.yaml b/families/roberta/tests/benchmark/roberta-base.yaml new file mode 100644 index 0000000000..98885944de --- /dev/null +++ b/families/roberta/tests/benchmark/roberta-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: roberta-base +candidate: + family: roberta + checkpoint: FacebookAI/roberta-base + revision: e2da8e2f811d1448a5b465c236feacd80ffbac7b + task: encoding + precision: fp16 + bundle: roberta-base.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/roberta/tests/benchmark/roberta-large.yaml b/families/roberta/tests/benchmark/roberta-large.yaml new file mode 100644 index 0000000000..4cafecb290 --- /dev/null +++ b/families/roberta/tests/benchmark/roberta-large.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: roberta-large +candidate: + family: roberta + checkpoint: FacebookAI/roberta-large + revision: 722cf37b1afa9454edce342e7895e588b6ff1d59 + task: encoding + precision: fp16 + bundle: roberta-large.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/roberta/tests/benchmark/xlm-roberta-base.yaml b/families/roberta/tests/benchmark/xlm-roberta-base.yaml new file mode 100644 index 0000000000..a53a2c68b0 --- /dev/null +++ b/families/roberta/tests/benchmark/xlm-roberta-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xlm-roberta-base +candidate: + family: roberta + checkpoint: FacebookAI/xlm-roberta-base + revision: e73636d4f797dec63c3081bb6ed5c7b0bb3f2089 + task: encoding + precision: fp16 + bundle: xlm-roberta-base.bundle + build: + max_sequence_length: 512 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/s1_mini/tests/benchmark/s1-mini-fp16.yaml b/families/s1_mini/tests/benchmark/s1-mini-fp16.yaml new file mode 100644 index 0000000000..d838b7ab98 --- /dev/null +++ b/families/s1_mini/tests/benchmark/s1-mini-fp16.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: s1-mini-fp16 +candidate: + family: s1_mini + checkpoint: superwhisper/s1-mini + revision: 65f84bcda1d13df582c4a8443c1c5aa53c0c66db + task: text_generation + precision: fp16 + bundle: s1-mini-fp16.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: mmlu-continuation + benchmark: mmlu_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: normalize-40 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: |- + [Styling: semi-formal] [Structure: prose] [Context: general] + so um i need to like send the the report by uh friday no wait make that thursday + max_new_tokens: 40 + use_chat_template: true + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/sam/tests/benchmark/reference.py b/families/sam/tests/benchmark/reference.py new file mode 100644 index 0000000000..a59f585c96 --- /dev/null +++ b/families/sam/tests/benchmark/reference.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers SAM reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import SamModel, SamProcessor + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = SamProcessor.from_pretrained(model_dir, use_fast=False) + model = SamModel.from_pretrained(model_dir, torch_dtype=dtype).eval().to("cuda") + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, request: Mapping[str, Any]) -> dict[str, Any]: + import torch + from PIL import Image + + image = Image.open(image_path).convert("RGB") + x = int(float(request.get("point_x", 0.5)) * image.width) + y = int(float(request.get("point_y", 0.5)) * image.height) + label = 1 if bool(request.get("is_foreground", True)) else 0 + encoded = reference.processor( + image, + input_points=[[[x, y]]], + input_labels=[[label]], + return_tensors="pt", + ) + return { + name: value.to( + device="cuda", + dtype=reference.dtype if value.is_floating_point() else value.dtype, + ) + for name, value in encoded.items() + if isinstance(value, torch.Tensor) + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any]) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + outputs = reference.model(**inputs) + masks = reference.processor.image_processor.post_process_masks( + outputs.pred_masks.cpu(), + inputs["original_sizes"].cpu(), + inputs["reshaped_input_sizes"].cpu(), + binarize=False, + )[0] + masks = masks.detach().float() + height, width = (int(value) for value in masks.shape[-2:]) + count = int(masks.numel() // (height * width)) + scores = outputs.iou_scores.detach().float().cpu().reshape(-1).tolist() + return { + "segmented_images": 1, + "generated_masks": count, + "num_masks": count, + "mask_pixels": int(masks.numel()), + "height": height, + "width": width, + "mask_kind": "logits", + "masks": masks.reshape(-1).tolist(), + "iou_scores": scores, + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + results = [] + for sample in samples: + inputs = _prepare(reference, str(sample["image_path"]), controls) + results.append({"sample_id": str(sample["sample_id"]), **_infer(reference, inputs)}) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "sam" or arguments.operation != "segment_prompted": + raise ValueError("SAM reference only supports sam segment_prompted") + if arguments.mode != "pytorch-eager": + raise ValueError("SAM reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), request) + for _ in range(arguments.warmup): + _infer(reference, inputs) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sam/tests/benchmark/sam-vit-base.yaml b/families/sam/tests/benchmark/sam-vit-base.yaml new file mode 100644 index 0000000000..c876d980d4 --- /dev/null +++ b/families/sam/tests/benchmark/sam-vit-base.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: sam-vit-base +candidate: + family: sam + checkpoint: facebook/sam-vit-base + revision: 70c1a07f894ebb5b307fd9eaaee97b9dfc16068f + task: prompted_segmentation + precision: fp16 + bundle: sam-vit-base.bundle + build: {} + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-prompted-mask-parity + benchmark: imagenette_prompted_segmentation_parity + samples: 10 + request: &prompted-request + point_x: 0.5 + point_y: 0.5 + is_foreground: true + reference: + command: reference.py + precision: fp32 + gate: + min_mask_iou: 0.7 + min_mask_match_rate: 1.0 + min_sample_pass_rate: 0.9 + +performance: + - name: segment-prompted + benchmark: semantic_segmentation_performance + operation: segment_prompted + request: + <<: *prompted-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: prompted-mask-parity + min_mask_iou: 0.7 diff --git a/families/sam3/tests/benchmark/reference.py b/families/sam3/tests/benchmark/reference.py new file mode 100644 index 0000000000..d2f2634141 --- /dev/null +++ b/families/sam3/tests/benchmark/reference.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers SAM3 reference for qualification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + processor: Any + model: Any + dtype: Any + + +def _load_reference(model_dir: Path, precision: str) -> Reference: + import torch + from transformers import Sam3Model, Sam3Processor + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + processor = Sam3Processor.from_pretrained(model_dir) + model = Sam3Model.from_pretrained(model_dir, torch_dtype=dtype).eval().to("cuda") + return Reference(processor, model, dtype) + + +def _prepare(reference: Reference, image_path: str, prompt: str) -> dict[str, Any]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + encoded = reference.processor(images=image, text=prompt, return_tensors="pt") + return { + name: value.to("cuda") if hasattr(value, "to") else value for name, value in encoded.items() + } + + +def _infer(reference: Reference, inputs: Mapping[str, Any]) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + outputs = reference.model(**inputs) + result = reference.processor.post_process_instance_segmentation( + outputs, + threshold=0.5, + mask_threshold=0.5, + target_sizes=inputs["original_sizes"].cpu().tolist(), + )[0] + masks = result["masks"].detach().float().cpu() + scores = result["scores"].detach().float().cpu().reshape(-1) + boxes = result["boxes"].detach().float().cpu().reshape(-1, 4) + if masks.ndim != 3 or masks.shape[0] != scores.numel() or boxes.shape[0] != scores.numel(): + raise RuntimeError("SAM3 reference returned inconsistent masks, scores or boxes") + height, width = (int(value) for value in masks.shape[-2:]) + count = int(masks.shape[0]) + return { + "segmented_images": 1, + "generated_masks": count, + "num_masks": count, + "mask_pixels": int(masks.numel()), + "height": height, + "width": width, + "mask_kind": "binary", + "masks": masks.reshape(-1).tolist(), + "iou_scores": scores.tolist(), + "boxes": boxes.tolist(), + "box_coordinates": "original_image_pixels_xyxy", + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + reference = _load_reference(model_dir, str(request.get("precision", "fp32"))) + controls = request.get("request", {}) + if not isinstance(controls, Mapping): + raise ValueError("reference request controls must be an object") + results = [] + for sample in samples: + prompt = str(sample.get("label_name", "")).strip() + if not prompt: + raise ValueError("SAM3 Accuracy sample has no label_name prompt") + inputs = _prepare(reference, str(sample["image_path"]), prompt) + results.append({"sample_id": str(sample["sample_id"]), **_infer(reference, inputs)}) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "sam3" or arguments.operation != "segment_prompted": + raise ValueError("SAM3 reference only supports sam3 segment_prompted") + if arguments.mode != "pytorch-eager": + raise ValueError("SAM3 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + reference = _load_reference(model_dir, arguments.precision) + inputs = _prepare(reference, str(request["image_path"]), str(request["prompt"])) + for _ in range(arguments.warmup): + _infer(reference, inputs) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _infer(reference, inputs) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "transformers", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": str(torch.__version__), + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sam3/tests/benchmark/sam3.yaml b/families/sam3/tests/benchmark/sam3.yaml new file mode 100644 index 0000000000..8ef47618a8 --- /dev/null +++ b/families/sam3/tests/benchmark/sam3.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: sam3 +candidate: + family: sam3 + checkpoint: facebook/sam3 + revision: 3c879f39826c281e95690f02c7821c4de09afae7 + task: text_prompted_segmentation + precision: fp32 + bundle: sam3.bundle + build: {} + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-text-instance-parity + benchmark: imagenette_text_prompted_instance_parity + samples: 10 + request: {} + reference: + command: reference.py + precision: fp32 + gate: + min_mask_iou: 0.7 + min_mask_match_rate: 1.0 + min_box_iou: 0.95 + max_score_abs_error: 0.05 + min_sample_pass_rate: 0.9 + +performance: + - name: segment-prompted + benchmark: text_prompted_segmentation_performance + operation: segment_prompted + request: + image_path: ../data/test_img.jpeg + prompt: car + measurement: + warmup: 3 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: transformers + precision: fp32 + output_contract: instance-mask-parity + min_mask_iou: 0.7 + min_box_iou: 0.95 + max_score_abs_error: 0.05 diff --git a/families/sana_wm/native_plugin_builder.py b/families/sana_wm/native_plugin_builder.py index 42ef7e2e78..321636fd99 100644 --- a/families/sana_wm/native_plugin_builder.py +++ b/families/sana_wm/native_plugin_builder.py @@ -6,6 +6,7 @@ from __future__ import annotations import ctypes +import importlib.util import subprocess import tempfile from pathlib import Path @@ -16,22 +17,52 @@ _PLUGIN_PATH: Path | None = None -def ensure_native_plugin(*, verbose: bool = False) -> Path: - """Build a fresh plugin library and return it.""" - import torch - - source_dir = Path(__file__).with_name("native_plugins") - build_dir = Path(tempfile.mkdtemp(prefix="sana-wm-plugin-")) - output = build_dir / "libtrtmc_sana_wm_native_plugin.so" - configure = [ +def _configure_command( + source_dir: Path, + build_dir: Path, + torch_prefix: str, + tensorrt_library: Path | None, +) -> list[str]: + command = [ "cmake", "-S", str(source_dir), "-B", str(build_dir), "-DCMAKE_BUILD_TYPE=Release", - f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", + f"-DCMAKE_PREFIX_PATH={torch_prefix}", ] + if tensorrt_library is not None: + command.append(f"-DSANA_WM_TRT_LIBRARY={tensorrt_library}") + return command + + +def _installed_tensorrt_library() -> Path | None: + """Return the TensorRT library shipped with the active Python installation.""" + + spec = importlib.util.find_spec("tensorrt_libs") + if spec is None: + return None + for location in spec.submodule_search_locations or (): + library = Path(location) / "libnvinfer.so.11" + if library.is_file(): + return library + return None + + +def ensure_native_plugin(*, verbose: bool = False) -> Path: + """Build a fresh plugin library and return it.""" + import torch + + source_dir = Path(__file__).with_name("native_plugins") + build_dir = Path(tempfile.mkdtemp(prefix="sana-wm-plugin-")) + output = build_dir / "libtrtmc_sana_wm_native_plugin.so" + configure = _configure_command( + source_dir, + build_dir, + torch.utils.cmake_prefix_path, + _installed_tensorrt_library(), + ) build = [ "cmake", "--build", @@ -40,7 +71,9 @@ def ensure_native_plugin(*, verbose: bool = False) -> Path: "trtmc_sana_wm_native_plugin", "-j2", ] - kwargs = {} if verbose else {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True} + kwargs = ( + {} if verbose else {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True} + ) try: subprocess.run(configure, check=True, **kwargs) subprocess.run(build, check=True, **kwargs) diff --git a/families/sana_wm/tests/benchmark/prepare_environment.py b/families/sana_wm/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..63e055b64c --- /dev/null +++ b/families/sana_wm/tests/benchmark/prepare_environment.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare pinned SANA source and a local HF-compatible model directory.""" + +from pathlib import Path +import os +import shutil +import subprocess +import sys +import tempfile + +from huggingface_hub import snapshot_download + +REPOSITORY = "https://github.com/NVlabs/Sana.git" +REVISION = "59629fdf790850797cb657bad014fce432bd713d" +MODEL = "Efficient-Large-Model/SANA-WM_bidirectional" +MODEL_REVISION = "e96271d77398def8ebb9fc595e7c0056dc625ab7" +STAGE1_TEXT_ENCODER = "Efficient-Large-Model/gemma-2-2b-it" +STAGE1_TEXT_ENCODER_REVISION = "569d9809d0c8b6722d4d31b5a77a2ec7a400650a" +OPENCV_HEADLESS_VERSION = "4.11.0.86" + + +def _install_headless_opencv() -> None: + # mmcv declares the GUI OpenCV distribution, whose Linux wheel requires + # libxcb. The reference only uses image-processing APIs, so replace it + # with the headless wheel inside this family environment. + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=False, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + f"opencv-python-headless=={OPENCV_HEADLESS_VERSION}", + ], + check=True, + ) + + +def _materialize_snapshot(snapshot: Path, destination: Path) -> None: + temporary = Path(tempfile.mkdtemp(prefix="SANA-model.", dir=destination.parent)) + try: + for source in snapshot.rglob("*"): + target = temporary / source.relative_to(snapshot) + if source.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + if not source.is_file(): + continue + target.parent.mkdir(parents=True, exist_ok=True) + resolved = source.resolve(strict=True) + try: + os.link(resolved, target) + except OSError: + shutil.copy2(resolved, target) + temporary.rename(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def _materialize_stage1_text_encoder(model: Path) -> None: + destination = model / "stage1_text_encoder" + required = ( + destination / "config.json", + destination / "model.safetensors.index.json", + destination / "tokenizer.json", + ) + if all(path.is_file() for path in required): + return + if destination.exists() or destination.is_symlink(): + raise RuntimeError(f"incomplete SANA stage-1 text encoder: {destination}") + snapshot = Path( + snapshot_download( + STAGE1_TEXT_ENCODER, + revision=STAGE1_TEXT_ENCODER_REVISION, + ) + ).resolve() + _materialize_snapshot(snapshot, destination) + + +def main() -> None: + _install_headless_opencv() + parent = Path(sys.prefix) / "trtmc-reference" + source = parent / "Sana" + parent.mkdir(parents=True, exist_ok=True) + if not (source / "inference_video_scripts/wm/inference_sana_wm.py").is_file(): + if source.exists(): + raise RuntimeError(f"incomplete SANA reference: {source}") + temporary = Path(tempfile.mkdtemp(prefix="Sana.", dir=parent)) + try: + subprocess.run(["git", "init", "--quiet", str(temporary)], check=True) + subprocess.run( + ["git", "-C", str(temporary), "remote", "add", "origin", REPOSITORY], check=True + ) + subprocess.run( + ["git", "-C", str(temporary), "fetch", "--quiet", "--depth=1", "origin", REVISION], + check=True, + ) + subprocess.run( + ["git", "-C", str(temporary), "checkout", "--quiet", "--detach", "FETCH_HEAD"], + check=True, + ) + temporary.rename(source) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + model = parent / "SANA-model" + if model.is_dir() and not model.is_symlink(): + _materialize_stage1_text_encoder(model) + return + if model.exists() or model.is_symlink(): + raise RuntimeError(f"incomplete SANA model directory: {model}") + snapshot = Path(snapshot_download(MODEL, revision=MODEL_REVISION)).resolve() + _materialize_snapshot(snapshot, model) + _materialize_stage1_text_encoder(model) + + +if __name__ == "__main__": + main() diff --git a/families/sana_wm/tests/benchmark/reference.py b/families/sana_wm/tests/benchmark/reference.py new file mode 100644 index 0000000000..3b46126303 --- /dev/null +++ b/families/sana_wm/tests/benchmark/reference.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the family-owned SANA-WM reference.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _execute( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> reference_harness.Measurement: + options = dict(options) + testcase_name = getattr(arguments, "testcase_name", None) + if testcase_name is not None: + manifest_value = json.loads(arguments.manifest.read_text(encoding="utf-8")) + matches = [ + value + for value in manifest_value.get("testcases", []) + if isinstance(value, Mapping) and value.get("name") == testcase_name + ] + if len(matches) != 1: + raise ValueError(f"SANA-WM requires exactly one testcase named {testcase_name!r}") + for name in ( + "translation_speed", + "rotation_speed_deg", + "fps", + "flow_shift", + "no_action_overlay", + ): + if name in matches[0]: + options.setdefault(name, matches[0][name]) + reference_repo = Path(str(options.get("reference_repo", ""))).resolve() + if not reference_repo.is_dir(): + raise ValueError("SANA-WM reference requires reference_repo") + model_root = arguments.manifest.resolve().parent.parent + + def model_input(value: Any) -> Path: + path = Path(str(value)) + return path.resolve() if path.is_absolute() else (model_root / path).resolve() + + image = model_input(request["image_path"]) + intrinsics = model_input( + options.get("intrinsics_path", options.get("intrinsics", "assets/demo_0_intrinsics.npy")) + ) + for label, path in (("image", image), ("intrinsics", intrinsics)): + if not path.is_file(): + raise FileNotFoundError(f"SANA-WM {label} input does not exist: {path}") + prompt_text = str(request.get("prompt", "")) + if not prompt_text: + raise ValueError("SANA-WM request requires prompt") + action = request.get("action") + if not isinstance(action, str) or not action: + raise ValueError("SANA-WM request requires a non-empty action") + if "model_dir" not in options: + raise ValueError("SANA-WM reference requires model_dir") + model_dir = Path(str(options["model_dir"])).resolve() + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + num_frames = int( + request["num_frames"] + if "num_frames" in request + else manifest["video_num_frames"] + ) + with tempfile.TemporaryDirectory(prefix="trtmc-perf-sana-wm-") as temporary: + root = Path(temporary) + output = root / "benchmark.json" + prompt = root / "prompt.txt" + prompt.write_text(prompt_text, encoding="utf-8") + command = [ + sys.executable, + str(Path(__file__).with_name("upstream_reference.py")), + "--reference-repo", + str(reference_repo), + "--image", + str(image), + "--model-dir", + str(model_dir), + "--prompt", + str(prompt), + "--action", + action, + "--intrinsics", + str(intrinsics), + "--translation_speed", + str( + request["translation_speed"] + if "translation_speed" in request + else options["translation_speed"] + ), + "--rotation_speed_deg", + str( + request["rotation_speed_deg"] + if "rotation_speed_deg" in request + else options["rotation_speed_deg"] + ), + "--num_frames", + str(num_frames), + "--fps", + str(request["fps"] if "fps" in request else options["fps"]), + "--step", + str(request["num_steps"]), + "--cfg_scale", + str(request["cfg_scale"]), + "--flow_shift", + str( + request["flow_shift"] if "flow_shift" in request else options["flow_shift"] + ), + "--seed", + str(request["seed"]), + "--refiner_seed", + str(request["seed"]), + "--warmup", + str(arguments.warmup), + "--iterations", + str(arguments.iterations), + "--output", + str(output), + ] + no_overlay = ( + request["no_action_overlay"] + if "no_action_overlay" in request + else options["no_action_overlay"] + ) + if bool(no_overlay): + command.append("--no_action_overlay") + completed = subprocess.run( + command, + cwd=reference_repo, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + raise RuntimeError( + f"SANA-WM reference failed with rc={completed.returncode}: " + f"{completed.stderr[-4000:]}" + ) + payload = json.loads(output.read_text(encoding="utf-8")) + summary = dict(payload.get("output_summary", {})) + artifacts = summary.get("frame_artifacts", []) + if artifacts: + destination = arguments.output.with_suffix(".media").resolve() + destination.mkdir(parents=True, exist_ok=True) + copied = [] + for source_value in artifacts: + source = Path(str(source_value)) + target = destination / source.name + shutil.copyfile(source, target) + copied.append(str(target)) + summary["frame_artifacts"] = copied + return reference_harness.Measurement( + [float(value) for value in payload.get("samples_ms", [])], + summary, + "sana-wm-pytorch", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + asset_loading_included=False, + ) + + +def _run_compat( + arguments: Any, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> tuple[list[float], dict[str, Any], str, str, bool, bool]: + """Compatibility shape for family-local unit tests written before the move.""" + value = _execute(arguments, request, options) + return ( + list(value.samples_ms), + dict(value.output_summary), + value.backend, + value.timing_scope, + value.input_preparation_included, + value.asset_loading_included, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + return reference_harness.run_premeasured( + list(sys.argv[1:] if argv is None else argv), + description=__doc__, + execute=_execute, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml new file mode 100644 index 0000000000..d7c4f20570 --- /dev/null +++ b/families/sana_wm/tests/benchmark/sana-wm-bidirectional.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: sana-wm-bidirectional +candidate: {family: sana_wm, checkpoint: Efficient-Large-Model/SANA-WM_bidirectional, revision: e96271d77398def8ebb9fc595e7c0056dc625ab7, model_directory: trtmc-reference/SANA-model, task: world_model_generation, precision: bf16, bundle: sana-wm-bidirectional.bundle, build: {max_sequence_length: 256, image_height: 704, image_width: 1280, video_num_frames: 321}} +reference_environment: + requirements: ../../requirements.txt + paths: {reference_repo: trtmc-reference/Sana, model_dir: trtmc-reference/SANA-model} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request + prompt_path: ../assets/demo_0.txt + image_path: ../assets/demo_0.png + action: w-80,jw-40,w-40,lw-60,w-100 + camera_intrinsics: [797.87866, 830.0503, 844.2675, 463.7225] + media_type: video + seed: 42 + num_steps: 60 + cfg_scale: 5.0 + height: 704 + width: 1280 + video_num_frames: 321 + translation_speed: 0.055 + rotation_speed_deg: 1.2 + fps: 16 + flow_shift: 9.8 + no_action_overlay: true + reference: &reference {script: tests/benchmark/reference.py, mode: pytorch-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {intrinsics_path: ../assets/demo_0_intrinsics.npy}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: official_sana, output_contract: media-shape}} diff --git a/apps/benchmark/performance/baselines/sana_wm_reference.py b/families/sana_wm/tests/benchmark/upstream_reference.py similarity index 83% rename from apps/benchmark/performance/baselines/sana_wm_reference.py rename to families/sana_wm/tests/benchmark/upstream_reference.py index 72d729e2f2..6d9f446bb8 100755 --- a/apps/benchmark/performance/baselines/sana_wm_reference.py +++ b/families/sana_wm/tests/benchmark/upstream_reference.py @@ -14,6 +14,15 @@ from typing import Any +REPOSITORY = Path(__file__).resolve().parents[4] +if str(REPOSITORY) not in sys.path: + sys.path.insert(0, str(REPOSITORY)) + +from families.sana_wm.tests.official_reference import ( # noqa: E402 + install_local_stage1_text_encoder, +) + + def parser() -> argparse.ArgumentParser: value = argparse.ArgumentParser() value.add_argument("--reference-repo", required=True, type=Path) @@ -57,6 +66,26 @@ def media_summary(video: Any) -> dict[str, int | str]: } +def materialize_media(video: Any, output: Path) -> dict[str, Any]: + import numpy as np + from PIL import Image + + frames = np.asarray(video) + summary: dict[str, Any] = media_summary(frames) + indices = sorted({0, len(frames) // 2, len(frames) - 1}) + directory = output.with_suffix(".media") + directory.mkdir(parents=True, exist_ok=True) + artifacts = [] + for index in indices: + values = frames[index] + scale = 255.0 if np.issubdtype(values.dtype, np.floating) else 1.0 + path = directory / f"{index:06d}.png" + Image.fromarray(np.clip(values * scale, 0, 255).astype(np.uint8)).convert("RGB").save(path) + artifacts.append(str(path.resolve())) + summary.update(artifact_indices=indices, frame_artifacts=artifacts) + return summary + + def _official_module(reference_repo: Path) -> Any: entrypoint = reference_repo / "inference_video_scripts/wm/inference_sana_wm.py" if not entrypoint.is_file(): @@ -82,6 +111,7 @@ def main() -> int: required_model_paths = { "config": model_dir / "config.yaml", "model": model_dir / "dit/sana_wm_1600m_720p.safetensors", + "stage1 text encoder": model_dir / "stage1_text_encoder", "refiner": model_dir / "refiner", "refiner text encoder": model_dir / "refiner/text_encoder", } @@ -126,6 +156,7 @@ def main() -> int: gemma_root=required_model_paths["refiner text encoder"], seed=arguments.refiner_seed, ) + install_local_stage1_text_encoder(official, required_model_paths["stage1 text encoder"]) pipeline = official.SanaWMPipeline( config=config, model_path=required_model_paths["model"], @@ -162,7 +193,7 @@ def invoke() -> Any: video = official.apply_overlay(video, result["c2w"]) arguments.output.write_text( json.dumps( - {"samples_ms": samples, "output_summary": media_summary(video)}, + {"samples_ms": samples, "output_summary": materialize_media(video, arguments.output)}, indent=2, sort_keys=True, ) diff --git a/families/sana_wm/tests/manifests/sana-wm-bidirectional.json b/families/sana_wm/tests/manifests/sana-wm-bidirectional.json index f998bc7c9a..53c1ccd8ff 100644 --- a/families/sana_wm/tests/manifests/sana-wm-bidirectional.json +++ b/families/sana_wm/tests/manifests/sana-wm-bidirectional.json @@ -3,7 +3,8 @@ "hf_id": "Efficient-Large-Model/SANA-WM_bidirectional", "hf_dependencies": [ { - "repo_id": "Efficient-Large-Model/gemma-2-2b-it" + "repo_id": "Efficient-Large-Model/gemma-2-2b-it", + "revision": "569d9809d0c8b6722d4d31b5a77a2ec7a400650a" } ], "bundle": "sana-wm-bidirectional.bundle", diff --git a/families/sana_wm/tests/official_reference.py b/families/sana_wm/tests/official_reference.py new file mode 100644 index 0000000000..4deee8437e --- /dev/null +++ b/families/sana_wm/tests/official_reference.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Launch the pinned official SANA-WM reference with local stage-1 assets.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys +from typing import Any, Sequence + + +_STAGE1_TEXT_ENCODER_NAME = "gemma-2-2b-it" + + +def _require_stage1_text_encoder(path: Path) -> Path: + resolved = path.resolve() + required = (resolved / "config.json", resolved / "tokenizer.json") + missing = [str(value) for value in required if not value.is_file()] + if not any(resolved.glob("model*.safetensors*")): + missing.append(str(resolved / "model*.safetensors*")) + if missing: + raise FileNotFoundError( + "SANA-WM local stage-1 text encoder is incomplete: " + ", ".join(missing) + ) + return resolved + + +def load_local_stage1_text_encoder(path: Path, device: str) -> tuple[Any, Any]: + """Load the exact prepared stage-1 tokenizer and decoder without Hub resolution.""" + + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + + resolved = _require_stage1_text_encoder(path) + tokenizer = AutoTokenizer.from_pretrained(str(resolved), local_files_only=True) + tokenizer.padding_side = "right" + decoder = ( + AutoModelForCausalLM.from_pretrained( + str(resolved), + local_files_only=True, + torch_dtype=torch.bfloat16, + ) + .get_decoder() + .to(device) + ) + return tokenizer, decoder + + +def install_local_stage1_text_encoder(official: Any, path: Path) -> None: + """Replace only the official pipeline's stage-1 asset locator.""" + + resolved = _require_stage1_text_encoder(path) + + def load(name: str = "T5", device: str = "cuda") -> tuple[Any, Any]: + if name != _STAGE1_TEXT_ENCODER_NAME: + raise ValueError( + "SANA-WM expected stage-1 text encoder " + f"{_STAGE1_TEXT_ENCODER_NAME!r}, found {name!r}" + ) + return load_local_stage1_text_encoder(resolved, device) + + official.get_tokenizer_and_text_encoder = load + + +def _official_module(reference_repository: Path) -> tuple[Path, Any]: + repository = reference_repository.resolve() + entrypoint = repository / "inference_video_scripts/wm/inference_sana_wm.py" + if not entrypoint.is_file(): + raise FileNotFoundError(f"official SANA-WM entrypoint does not exist: {entrypoint}") + if str(repository) not in sys.path: + sys.path.insert(0, str(repository)) + from inference_video_scripts.wm import inference_sana_wm + + return entrypoint, inference_sana_wm + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(add_help=False) + value.add_argument("--reference-repo", required=True, type=Path) + value.add_argument("--stage1-text-encoder", required=True, type=Path) + return value + + +def main(argv: Sequence[str] | None = None) -> int: + arguments, official_arguments = parser().parse_known_args(argv) + entrypoint, official = _official_module(arguments.reference_repo) + install_local_stage1_text_encoder(official, arguments.stage1_text_encoder) + previous = sys.argv + try: + sys.argv = [str(entrypoint), *official_arguments] + result = official.main() + finally: + sys.argv = previous + return int(result or 0) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/sana_wm/tests/test_benchmark_reference.py b/families/sana_wm/tests/test_benchmark_reference.py new file mode 100644 index 0000000000..37db469b73 --- /dev/null +++ b/families/sana_wm/tests/test_benchmark_reference.py @@ -0,0 +1,673 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace +import json +from pathlib import Path +import sys +from types import ModuleType, SimpleNamespace + +import numpy as np +import pytest +import yaml + +from families.sana_wm.tests.benchmark import reference as sana_family_reference +from families.sana_wm.tests.benchmark import upstream_reference as sana_wm_reference +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance import reference_harness +from qualification_tests.benchmark_qualification.performance.references import generic_reference + + +REPO = Path(__file__).resolve().parents[3] +SUITE = REPO / "qualification_tests/benchmark_qualification/performance/config/release.yaml" + + +def _environment(tmp_path: Path) -> tuple[Path, perf.Environment]: + tools = tmp_path / "tools" + tools.mkdir() + for name in ("bench", "worker", "hf.py", "task.py"): + path = tools / name + path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + path.chmod(0o755) + runtime = tmp_path / "runtime" + runtime.mkdir() + for name in ( + "libtrtmc_runtime.so", + "libtrtmc_backend_trt.so", + "libtrtmc_model_gpt2.so", + "libtrtmc_model_lance.so", + ): + (runtime / name).write_bytes(b"") + _, entries, _ = perf._load_suite_file(SUITE) + reference_fields = { + str(declaration["environment"]) + for entry in entries + for declaration in entry.get("baseline", {}).get("reference_inputs", {}).values() + } + references = {} + for name in reference_fields: + path = tmp_path / name + path.mkdir() + references[name] = str(path) + value = { + "schema_version": perf.ENVIRONMENT_SCHEMA, + "name": "test", + "tools": { + "trtmc_bench": str(tools / "bench"), + "trtmc_worker": str(tools / "worker"), + "hf_transformers_runner": str(tools / "hf.py"), + "task_reference_runner": str(tools / "task.py"), + }, + "references": references, + "storage": { + "results_root": str(tmp_path / "results"), + "scratch_root": str(tmp_path / "scratch"), + "bundle_cache": str(tmp_path / "bundles"), + "bundle_roots": [], + "runtime_root": str(runtime), + "bundle_retention": "retain", + }, + "execution": {"local_files_only": True, "timeout_seconds": 10}, + } + path = tmp_path / "environment.yaml" + path.write_text(yaml.safe_dump(value), encoding="utf-8") + return path, perf.load_environment(path) + + +def test_sana_reference_reports_materialized_video_shape() -> None: + video = np.stack( + [ + np.zeros((24, 32, 3), dtype=np.uint8), + np.ones((24, 32, 3), dtype=np.uint8), + ] + ) + summary = sana_wm_reference.media_summary(video) + assert summary == { + "media_type": "video", + "media_count": 2, + "num_frames": 2, + "height": 24, + "width": 32, + "channels": 3, + } + assert perf._media_shape(summary) == (2, 24, 32, 3) + source = (REPO / "families/sana_wm/tests/benchmark/reference.py").read_text( + encoding="utf-8" + ) + assert 'request.get("action"' in source + + +def test_sana_world_request_preserves_official_camera_controls(tmp_path: Path) -> None: + model = replace( + perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional"), + task="world_model_generation", + ) + request = perf.resolve_case( + model, tmp_path / "model.bundle", selected_task="world_model_generation", + ).request + assert request["translation_speed"] == 0.055 + assert request["rotation_speed_deg"] == 1.2 + assert request["fps"] == 16 + assert request["flow_shift"] == 9.8 + assert request["no_action_overlay"] is True + + +@pytest.mark.parametrize("route", ["builtin", "script", "hf"]) +def test_reference_testcase_name_is_only_sent_to_builtin_runner(monkeypatch, tmp_path, route): + _, environment = _environment(tmp_path) + # Exercise explicit runner fixtures, not a family's active owned replacement. + _, entries, _ = perf._load_suite_file(SUITE) + entry = perf.resolve_entries( + [value for value in entries if value["id"] == "sana_wm.generate_image"], environment, + )[0] + baseline = dict(entry.spec["baseline"]) + if route == "builtin": + baseline.pop("script") + baseline["adapter"] = "hf-diffusers" + elif route == "script": + baseline["script"] = "tests/reference.py" + monkeypatch.setattr(perf, "_family_script", lambda _: tmp_path / "reference.py") + elif route == "hf": + baseline["runner"] = "hf-transformers" + entry = replace(entry, spec={**entry.spec, "baseline": baseline}) + command = perf.baseline_command(entry, environment, tmp_path / "reference.json") + assert command[command.index("--case-name") + 1] == entry.spec["id"] + if route == "builtin": + assert command[command.index("--testcase-name") + 1] == entry.case.testcase_name + parser = generic_reference.build_parser() + parsed = parser.parse_args(command[2:]) + assert parsed.testcase_name == entry.case.testcase_name + index = command.index("--testcase-name") + direct = command[2:index] + command[index + 2:] + assert parser.parse_args(direct).testcase_name is None + else: + assert "--testcase-name" not in command + + +@pytest.mark.parametrize("selection", ["selected", "missing", "duplicate", ""]) +def test_sana_reference_resolves_only_exact_testcase_before_execution( + monkeypatch, tmp_path, selection, +): + manifest = tmp_path / "manifest.json" + controls = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + chosen = {"name": "selected", **controls} + manifest.write_text(json.dumps({"video_num_frames": 321, "testcases": [ + {"name": "performance-entry", **dict.fromkeys(controls, "wrong testcase")}, + chosen, *([chosen] if selection == "duplicate" else []), + ]})) + arguments = SimpleNamespace( + manifest=manifest, testcase_name="selected" if selection == "duplicate" else selection, + case_name="performance-entry", warmup=1, iterations=2, + ) + assets = REPO / "families/sana_wm/tests/assets" + request = {"prompt": "drive forward", "image_path": str(assets / "demo_0.png"), + "action": "w-320", "num_steps": 60, "cfg_scale": 5.0, "seed": 42} + options = {"reference_repo": str(tmp_path), "model_dir": str(tmp_path / "model"), + "intrinsics": str(assets / "demo_0_intrinsics.npy")} + before = deepcopy(request), deepcopy(options), manifest.read_bytes() + + def run(command, **kwargs): + assert selection == "selected", "invalid selection must fail before reference execution" + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(controls[name]) + assert "--no_action_overlay" in command + assert command[command.index("--num_frames") + 1] == "321" + raise RuntimeError("captured selected reference command") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + if selection == "selected": + with pytest.raises(RuntimeError, match="captured selected reference command"): + sana_family_reference._run_compat(arguments, request, options) + else: + with pytest.raises(ValueError, match="exactly one testcase"): + sana_family_reference._run_compat(arguments, request, options) + assert (request, options, manifest.read_bytes()) == before + + +def test_sana_reference_options_use_resolved_testcase_and_explicit_options( + monkeypatch, tmp_path: Path, +) -> None: + _, environment = _environment(tmp_path) + # This tests the existing builtin adapter even after an owned script takes over. + _, entries, _ = perf._load_suite_file(SUITE) + selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] + entry = perf.resolve_entries(selected, environment)[0] + original = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + assert entry.case.testcase_name != entry.spec["id"] + testcase = next(value for value in entry.manifest["testcases"] + if value["name"] == entry.case.testcase_name) + other = {**testcase, "name": entry.spec["id"], **dict.fromkeys(original, "wrong testcase")} + entry = replace(entry, manifest={**entry.manifest, "testcases": [other, testcase]}) + semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") + request = generic_reference.flatten_config(semantic.request) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(entry.manifest)) + arguments = SimpleNamespace( + manifest=manifest_path, testcase_name=entry.case.testcase_name, warmup=1, iterations=2, + ) + before = json.dumps(entry.manifest, sort_keys=True) + options = perf._adapter_options(entry, environment) + assert {name: options[name] for name in original} == original + assert "action" not in options and "prompt" not in options + assert options["reference_repo"] == str(Path(environment.references["sana_repo"]).resolve()) + assert options["model_dir"] == str(Path(environment.references["sana_model"]).resolve()) + options["intrinsics"] = str(entry.model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy") + expected = original + + def run(command, **kwargs): + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(expected[name]) + assert ("--no_action_overlay" in command) is expected["no_action_overlay"] + assert command[command.index("--action") + 1] == request["action"] + assert Path(command[command.index("--prompt") + 1]).read_text() == request["prompt"] + raise RuntimeError("captured reference options") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + original_options = deepcopy(options) + with pytest.raises(RuntimeError, match="captured reference options"): + sana_family_reference._run_compat(arguments, request, options) + assert options == original_options + explicit = {"translation_speed": 0.0, "rotation_speed_deg": 0.0, + "fps": 0, "flow_shift": 0.0, "no_action_overlay": False} + configured = {**entry.spec["baseline"].get("adapter_options", {}), **explicit} + entry = replace(entry, spec={**entry.spec, "baseline": {**entry.spec["baseline"], "adapter_options": configured}}) + options = perf._adapter_options(entry, environment) + assert {name: options[name] for name in original} == explicit + assert options["no_action_overlay"] is False + options["intrinsics"] = original_options["intrinsics"] + explicit_options = deepcopy(options) + expected = explicit + with pytest.raises(RuntimeError, match="captured reference options"): + sana_family_reference._run_compat(arguments, request, options) + assert options == explicit_options + assert request == generic_reference.flatten_config(semantic.request) + assert json.loads(manifest_path.read_text()) == entry.manifest + assert json.dumps(entry.manifest, sort_keys=True) == before + assert entry.spec["baseline"]["adapter_options"] == configured + + +def test_sana_semantic_request_metadata_moves_only_to_reference_options( + monkeypatch, tmp_path: Path, +) -> None: + _, environment = _environment(tmp_path) + # Keep the builtin reference contract independent of canonical script selection. + _, entries, _ = perf._load_suite_file(SUITE) + selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] + entry = perf.resolve_entries(selected, environment)[0] + semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") + entry = replace(entry, case=semantic) + options = perf._adapter_options(entry, environment) + expected = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + assert not expected.keys() & semantic.request.keys() + assert {name: options[name] for name in expected} == expected + command = perf.baseline_command(entry, environment, tmp_path / "reference.json") + arguments = reference_harness.parser("test").parse_args(command[2:]) + + def run(command, **kwargs): + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(expected[name]) + assert "--no_action_overlay" in command + raise RuntimeError("captured reference metadata") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + with pytest.raises(RuntimeError, match="captured reference metadata"): + sana_family_reference._run_compat( + arguments, generic_reference.flatten_config(semantic.request), options + ) + assert semantic.request["action"] == entry.manifest["testcases"][0]["action"] + assert semantic.request["seed"] == 42 and semantic.request["num_steps"] == 60 + assert semantic.request["cfg_scale"] == 5.0 + + +def test_sana_reference_calls_official_pipeline_with_exact_workload( + monkeypatch, tmp_path: Path +) -> None: + captured: dict[str, object] = {"generate": []} + reference_repo = tmp_path / "Sana" + reference_repo.mkdir() + model_dir = tmp_path / "model" + (model_dir / "dit").mkdir(parents=True) + (model_dir / "refiner/text_encoder").mkdir(parents=True) + stage1_text_encoder = model_dir / "stage1_text_encoder" + stage1_text_encoder.mkdir() + for name in ("config.json", "tokenizer.json", "model.safetensors"): + (stage1_text_encoder / name).write_text("{}\n", encoding="utf-8") + (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") + (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") + image = tmp_path / "image.png" + image.write_bytes(b"image") + prompt = tmp_path / "prompt.txt" + prompt.write_text("drive forward", encoding="utf-8") + intrinsics = tmp_path / "intrinsics.npy" + intrinsics.write_bytes(b"intrinsics") + output = tmp_path / "result.json" + + class FakeImage: + def convert(self, mode): + captured["image_mode"] = mode + return self + + def save(self, path): + Path(path).write_bytes(b"png") + return None + + pil = ModuleType("PIL") + pil.Image = SimpleNamespace(open=lambda path: FakeImage(), fromarray=lambda value: FakeImage()) + monkeypatch.setitem(sys.modules, "PIL", pil) + + synchronize_calls = [] + torch = ModuleType("torch") + torch.cuda = SimpleNamespace( + is_available=lambda: True, + synchronize=lambda: synchronize_calls.append(True), + ) + monkeypatch.setitem(sys.modules, "torch", torch) + + pyrallis = ModuleType("pyrallis") + + def parse_config(**kwargs): + captured["parse"] = kwargs + return "config" + + pyrallis.parse = parse_config + monkeypatch.setitem(sys.modules, "pyrallis", pyrallis) + + class RefinerSettings: + def __init__(self, **kwargs): + captured["refiner"] = kwargs + + class GenerationParams: + def __init__(self, **kwargs): + captured["generation"] = kwargs + + class Pipeline: + def __init__(self, **kwargs): + captured["pipeline"] = kwargs + + def generate(self, *args): + captured["generate"].append(args) + return { + "video": np.zeros((321, 24, 32, 3), dtype=np.uint8), + "c2w": "camera", + } + + trajectory = np.zeros((321, 4, 4), dtype=np.float32) + official = SimpleNamespace( + InferenceConfig=object, + RefinerSettings=RefinerSettings, + GenerationParams=GenerationParams, + SanaWMPipeline=Pipeline, + action_string_to_c2w=lambda action, **kwargs: ( + captured.update(action=(action, kwargs)) or trajectory + ), + _snap_num_frames=lambda value, **kwargs: value, + resize_and_center_crop=lambda value: ("cropped", (1, 1), (2, 2), (0, 0)), + load_intrinsics=lambda path, frames: ( + captured.update(load_intrinsics=(path, frames)) or "raw-intrinsics" + ), + transform_intrinsics_for_crop=lambda value, *sizes: ( + captured.update(transform_intrinsics=(value, sizes)) or "intrinsics" + ), + apply_overlay=lambda *_: (_ for _ in ()).throw( + AssertionError("no-action-overlay must skip overlay") + ), + ) + monkeypatch.setattr(sana_wm_reference, "_official_module", lambda path: official) + installed = {} + monkeypatch.setattr( + sana_wm_reference, + "install_local_stage1_text_encoder", + lambda module, path: installed.update(module=module, path=path), + ) + monkeypatch.setattr( + sys, + "argv", + [ + "sana_wm_reference.py", + "--reference-repo", + str(reference_repo), + "--image", + str(image), + "--model-dir", + str(model_dir), + "--prompt", + str(prompt), + "--action", + "w-320", + "--intrinsics", + str(intrinsics), + "--num_frames", + "321", + "--fps", + "16", + "--step", + "60", + "--cfg_scale", + "5.0", + "--flow_shift", + "9.8", + "--seed", + "42", + "--refiner_seed", + "42", + "--translation_speed", + "0.055", + "--rotation_speed_deg", + "1.2", + "--no_action_overlay", + "--warmup", + "1", + "--iterations", + "2", + "--output", + str(output), + ], + ) + + assert sana_wm_reference.main() == 0 + assert captured["action"] == ( + "w-320", + {"translation_speed": 0.055, "rotation_speed_deg": 1.2}, + ) + assert captured["refiner"] == { + "root": model_dir / "refiner", + "gemma_root": model_dir / "refiner/text_encoder", + "seed": 42, + } + assert installed == {"module": official, "path": stage1_text_encoder.resolve()} + assert captured["generation"] == { + "num_frames": 321, + "fps": 16, + "step": 60, + "cfg_scale": 5.0, + "flow_shift": 9.8, + "seed": 42, + } + assert len(captured["generate"]) == 3 + assert len(synchronize_calls) == 5 + result = json.loads(output.read_text(encoding="utf-8")) + assert result["output_summary"]["num_frames"] == 321 + + +def test_sana_reference_requires_action_from_current_request(tmp_path: Path) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json" + ) + with pytest.raises(ValueError, match="non-empty action"): + sana_family_reference._run_compat( + arguments, + {"prompt": "drive", "image_path": "assets/demo_0.png"}, + {"reference_repo": str(checkout)}, + ) + + +@pytest.mark.parametrize("frame_controls", [{"num_frames": 321}, {}]) +def test_sana_generic_reference_uses_one_explicit_official_command( + monkeypatch, tmp_path: Path, frame_controls: dict, +) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + model_dir = tmp_path / "model" + model_dir.mkdir() + captured = {} + + def run(command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples_ms": [1.0, 2.0], + "output_summary": { + "media_type": "video", + "media_count": 321, + "num_frames": 321, + "height": 704, + "width": 1280, + "channels": 3, + }, + } + ), + encoding="utf-8", + ) + return SimpleNamespace(returncode=0, stderr="") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", + warmup=1, + iterations=2, + ) + result = sana_family_reference._run_compat( + arguments, + { + "prompt": "drive forward", + "image_path": "assets/demo_0.png", + "action": "w-80,jw-40,w-40,lw-60,w-100", + "translation_speed": 0.055, + "rotation_speed_deg": 1.2, + **frame_controls, + "fps": 16, + "num_steps": 60, + "cfg_scale": 5.0, + "flow_shift": 9.8, + "seed": 42, + "no_action_overlay": True, + }, + { + "reference_repo": str(checkout), + "model_dir": str(model_dir), + "intrinsics": "assets/demo_0_intrinsics.npy", + }, + ) + + command = captured["command"] + assert command[command.index("--reference-repo") + 1] == str(checkout.resolve()) + assert command[command.index("--translation_speed") + 1] == "0.055" + assert command[command.index("--rotation_speed_deg") + 1] == "1.2" + assert command[command.index("--num_frames") + 1] == "321" + assert command[command.index("--fps") + 1] == "16" + assert command[command.index("--flow_shift") + 1] == "9.8" + assert command[command.index("--refiner_seed") + 1] == "42" + assert command[command.index("--warmup") + 1] == "1" + assert command[command.index("--iterations") + 1] == "2" + assert "--no_action_overlay" in command + assert "env" not in captured["kwargs"] + assert result[0] == [1.0, 2.0] + assert result[1]["num_frames"] == 321 + + +@pytest.mark.parametrize("controls,manifest_frames,expected", [ + ({}, 321, "321"), + ({"num_frames": 7}, 321, "7"), + ({"num_frames": 0}, 321, "0"), + ({"config": {"num_frames": 0}}, 321, "0"), + ({"num_frames": 7}, None, "7"), + ({}, None, None), +]) +def test_sana_semantic_reference_uses_manifest_frames_only_when_absent( + monkeypatch, tmp_path: Path, controls: dict, manifest_frames: int | None, expected: str | None, +) -> None: + model = perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional") + model = replace(model, testcases=({**model.testcases[0], **controls},)) + case = perf.resolve_case(model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") + request = generic_reference.flatten_config(case.request) + before = dict(request) + manifest = json.loads(model.manifest_path.read_text()) + if manifest_frames is None: + del manifest["video_num_frames"] + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest)) + checkout = tmp_path / "Sana" + checkout.mkdir() + options = { + "reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), + "intrinsics": str(model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy"), + } + testcase = next(value for value in manifest["testcases"] if value["name"] == case.testcase_name) + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay"): + options[name] = testcase[name] + assert name not in request + + def run(command, **kwargs): + assert command[command.index("--num_frames") + 1] == expected + assert command[command.index("--action") + 1] == request["action"] + assert command[command.index("--translation_speed") + 1] == "0.055" + assert command[command.index("--rotation_speed_deg") + 1] == "1.2" + assert command[command.index("--fps") + 1] == "16" + assert command[command.index("--flow_shift") + 1] == "9.8" + assert "--no_action_overlay" in command + raise RuntimeError("captured reference command") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + arguments = SimpleNamespace(manifest=manifest_path, warmup=1, iterations=2) + if expected is None: + with pytest.raises(KeyError, match="video_num_frames"): + sana_family_reference._run_compat(arguments, request, options) + else: + with pytest.raises(RuntimeError, match="captured reference command"): + sana_family_reference._run_compat(arguments, request, options) + assert request == before + + +@pytest.mark.parametrize("source", ["request", "config", "options"]) +@pytest.mark.parametrize("controls", [ + {"translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True}, + {"translation_speed": 0.0, "rotation_speed_deg": 0.0, + "fps": 0, "flow_shift": 0.0, "no_action_overlay": False}, +]) +def test_sana_reference_control_presence_and_request_precedence( + monkeypatch, tmp_path: Path, source: str, controls: dict, +) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", + warmup=1, iterations=2, + ) + request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", + "action": "w-80,jw-40,w-40,lw-60,w-100", "num_frames": 321, + "num_steps": 60, "cfg_scale": 5.0, "seed": 42} + options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), + "translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + if source == "options": + options.update(controls) + elif source == "config": + request = generic_reference.flatten_config({**request, "config": controls}) + else: + request.update(controls) + before = dict(request), dict(options) + + def run(command, **kwargs): + for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): + assert command[command.index("--" + name) + 1] == str(controls[name]) + assert ("--no_action_overlay" in command) is controls["no_action_overlay"] + assert command[command.index("--action") + 1] == request["action"] + assert command[command.index("--refiner_seed") + 1] == "42" + assert "env" not in kwargs + raise RuntimeError("captured reference controls") + + monkeypatch.setattr(sana_family_reference.subprocess, "run", run) + with pytest.raises(RuntimeError, match="captured reference controls"): + sana_family_reference._run_compat(arguments, request, options) + assert (request, options) == before + + +@pytest.mark.parametrize("missing", [ + "translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay", +]) +def test_sana_reference_control_missing_from_request_and_options_still_errors( + monkeypatch, tmp_path: Path, missing: str, +) -> None: + checkout = tmp_path / "Sana" + checkout.mkdir() + arguments = SimpleNamespace( + manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", + warmup=1, iterations=2, + ) + request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", "action": "w-320", + "num_frames": 321, "num_steps": 60, "cfg_scale": 5.0, "seed": 42} + options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), + "translation_speed": 0.055, "rotation_speed_deg": 1.2, + "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} + del options[missing] + monkeypatch.setattr( + sana_family_reference.subprocess, + "run", + lambda *args, **kwargs: pytest.fail( + "missing control must fail before reference execution" + ), + ) + with pytest.raises(KeyError, match=missing): + sana_family_reference._run_compat(arguments, request, options) diff --git a/families/sana_wm/tests/test_e2e.py b/families/sana_wm/tests/test_e2e.py index 55102c23ff..66f8f2fe37 100644 --- a/families/sana_wm/tests/test_e2e.py +++ b/families/sana_wm/tests/test_e2e.py @@ -141,8 +141,14 @@ def _prepared_model_dir(model_dir: Path, manifest: dict, tmp_path: Path) -> Path stage1_text_encoder = Path( snapshot_download( repo_id=stage1_repo, + revision=dependencies[0].get("revision"), local_files_only=True, - allow_patterns=["config.json"], + allow_patterns=[ + "config.json", + "model*.safetensors*", + "special_tokens_map.json", + "tokenizer*", + ], ) ) prepared = tmp_path / "prepared-model" @@ -327,12 +333,18 @@ def _official_reference(model_dir: Path, manifest: dict, case: dict, tmp_path: P model_dir = model_dir.resolve() entrypoint = source_root / "inference_video_scripts/wm/inference_sana_wm.py" assert entrypoint.is_file(), f"declared Sana reference entrypoint is missing: {entrypoint}" + launcher = TEST_ROOT / "official_reference.py" + assert launcher.is_file(), f"Sana family reference launcher is missing: {launcher}" config = model_dir / "config.yaml" model_path = model_dir / "dit/sana_wm_1600m_720p.safetensors" + stage1_text_encoder = model_dir / "stage1_text_encoder" refiner_root = model_dir / "refiner" refiner_gemma_root = refiner_root / "text_encoder" assert config.is_file(), f"Sana reference config is missing: {config}" assert model_path.is_file(), f"Sana reference weights are missing: {model_path}" + assert stage1_text_encoder.is_dir(), ( + f"Sana reference stage-1 text encoder is missing: {stage1_text_encoder}" + ) assert refiner_root.is_dir(), f"Sana reference refiner is missing: {refiner_root}" assert refiner_gemma_root.is_dir(), ( f"Sana reference refiner text encoder is missing: {refiner_gemma_root}" @@ -343,7 +355,11 @@ def _official_reference(model_dir: Path, manifest: dict, case: dict, tmp_path: P environment["PYTHONPATH"] = str(source_root) command = [ sys.executable, - str(entrypoint), + str(launcher), + "--reference-repo", + str(source_root), + "--stage1-text-encoder", + str(stage1_text_encoder), "--image", str(_asset(case["test_image"], report_key="image")), "--prompt", @@ -538,13 +554,14 @@ def test_official_checkpoint_e2e(case_name: str, tmp_path: Path) -> None: record_evidence("checkpoint", {"model_dir": str(model_dir), "hf_id": manifest.get("hf_id"), "hf_revision": manifest.get("hf_revision")}) binary, runtime_root = _runtime(manifest) bundle = tmp_path / manifest["bundle"] + prepared_model_dir = _prepared_model_dir(model_dir, manifest, tmp_path) with evidence_stage("build"): - _build(_prepared_model_dir(model_dir, manifest, tmp_path), bundle, manifest) + _build(prepared_model_dir, bundle, manifest) with evidence_stage("native"): actual = _native(binary, runtime_root, bundle, model_dir, manifest, case, tmp_path) record_evidence("native", native_snapshot(actual)) with evidence_stage("reference"): - expected = _official_reference(model_dir, manifest, case, tmp_path) + expected = _official_reference(prepared_model_dir, manifest, case, tmp_path) record_report_views(actual, expected, tmp_path / "paired-report-views") record_evidence("reference", reference_snapshot(expected)) with evidence_stage("compare"): diff --git a/families/sana_wm/tests/test_reference_contract.py b/families/sana_wm/tests/test_reference_contract.py index bc92d94003..761c96a8f8 100644 --- a/families/sana_wm/tests/test_reference_contract.py +++ b/families/sana_wm/tests/test_reference_contract.py @@ -5,11 +5,65 @@ import json from pathlib import Path +import sys +from types import ModuleType import numpy as np import pytest +import yaml +from .. import native_plugin_builder +from . import official_reference from . import test_e2e as e2e +from .benchmark import prepare_environment + + +def _write_stage1_text_encoder(model_dir: Path) -> Path: + stage1_text_encoder = model_dir / "stage1_text_encoder" + stage1_text_encoder.mkdir(parents=True) + for name in ("config.json", "tokenizer.json", "model.safetensors"): + (stage1_text_encoder / name).write_text("{}\n", encoding="utf-8") + return stage1_text_encoder + + +def test_qualification_candidate_uses_the_prepared_family_model() -> None: + profile = yaml.safe_load( + (Path(__file__).parent / "benchmark/sana-wm-bidirectional.yaml").read_text(encoding="utf-8") + ) + + prepared_model = "trtmc-reference/SANA-model" + assert profile["candidate"]["model_directory"] == prepared_model + assert profile["reference_environment"]["paths"]["model_dir"] == prepared_model + + +def test_native_plugin_uses_installed_tensorrt_library(tmp_path: Path) -> None: + tensorrt_library = tmp_path / "tensorrt/libnvinfer.so.11" + command = native_plugin_builder._configure_command( + tmp_path / "source", + tmp_path / "build", + "/torch/cmake", + tensorrt_library, + ) + + assert f"-DSANA_WM_TRT_LIBRARY={tensorrt_library}" in command + + +def test_native_plugin_discovers_tensorrt_python_library( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + package = tmp_path / "tensorrt_libs" + package.mkdir() + library = package / "libnvinfer.so.11" + library.touch() + spec = native_plugin_builder.importlib.util.spec_from_file_location( + "tensorrt_libs", + package / "__init__.py", + submodule_search_locations=[str(package)], + ) + monkeypatch.setattr(native_plugin_builder.importlib.util, "find_spec", lambda _name: spec) + + assert native_plugin_builder._installed_tensorrt_library() == library def test_official_source_dependencies_are_family_owned() -> None: @@ -29,6 +83,131 @@ def test_official_source_dependencies_are_family_owned() -> None: } <= requirements +def test_qualification_snapshot_is_materialized_inside_family_environment( + tmp_path: Path, +) -> None: + cache = tmp_path / "cache" + blob = cache / "blobs/weights" + blob.parent.mkdir(parents=True) + blob.write_bytes(b"weights") + snapshot = cache / "snapshots/revision" + (snapshot / "dit").mkdir(parents=True) + (snapshot / "dit/model.safetensors").symlink_to(blob) + environment = tmp_path / "environment" + environment.mkdir() + destination = environment / "SANA-model" + + prepare_environment._materialize_snapshot(snapshot, destination) + + materialized = destination / "dit/model.safetensors" + assert destination.is_dir() and not destination.is_symlink() + assert destination.resolve().is_relative_to(environment) + assert materialized.read_bytes() == b"weights" + assert materialized.samefile(blob) + assert prepare_environment.MODEL_REVISION == "e96271d77398def8ebb9fc595e7c0056dc625ab7" + + +def test_qualification_materializes_pinned_stage1_text_encoder(monkeypatch, tmp_path: Path) -> None: + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + for name in ("config.json", "model.safetensors.index.json", "tokenizer.json"): + (snapshot / name).write_text("{}\n", encoding="utf-8") + model = tmp_path / "model" + model.mkdir() + captured = {} + + def download(model_id, **kwargs): + captured.update(model_id=model_id, **kwargs) + return str(snapshot) + + monkeypatch.setattr(prepare_environment, "snapshot_download", download) + prepare_environment._materialize_stage1_text_encoder(model) + + destination = model / "stage1_text_encoder" + assert destination.is_dir() and not destination.is_symlink() + assert (destination / "tokenizer.json").is_file() + assert captured == { + "model_id": "Efficient-Large-Model/gemma-2-2b-it", + "revision": "569d9809d0c8b6722d4d31b5a77a2ec7a400650a", + } + + +def test_official_reference_loads_stage1_encoder_from_explicit_local_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + stage1_text_encoder = tmp_path / "stage1-text-encoder" + stage1_text_encoder.mkdir() + for name in ("config.json", "tokenizer.json", "model.safetensors"): + (stage1_text_encoder / name).write_text("{}\n", encoding="utf-8") + calls: list[tuple[object, ...]] = [] + + class Decoder: + def to(self, device: str): + calls.append(("to", device)) + return "local-decoder" + + class Model: + def get_decoder(self): + calls.append(("get_decoder",)) + return Decoder() + + class AutoTokenizer: + padding_side = "left" + + @staticmethod + def from_pretrained(path: str, **kwargs): + calls.append(("tokenizer", path, kwargs)) + return AutoTokenizer() + + class AutoModelForCausalLM: + @staticmethod + def from_pretrained(path: str, **kwargs): + calls.append(("model", path, kwargs)) + return Model() + + transformers = ModuleType("transformers") + transformers.AutoModelForCausalLM = AutoModelForCausalLM + transformers.AutoTokenizer = AutoTokenizer + torch = ModuleType("torch") + torch.bfloat16 = "bf16" + monkeypatch.setitem(sys.modules, "transformers", transformers) + monkeypatch.setitem(sys.modules, "torch", torch) + + tokenizer, decoder = official_reference.load_local_stage1_text_encoder( + stage1_text_encoder, "cuda:0" + ) + assert isinstance(tokenizer, AutoTokenizer) + assert tokenizer.padding_side == "right" + assert decoder == "local-decoder" + assert calls == [ + ("tokenizer", str(stage1_text_encoder), {"local_files_only": True}), + ( + "model", + str(stage1_text_encoder), + {"local_files_only": True, "torch_dtype": "bf16"}, + ), + ("get_decoder",), + ("to", "cuda:0"), + ] + + +def test_qualification_environment_replaces_gui_opencv(monkeypatch) -> None: + calls = [] + monkeypatch.setattr( + prepare_environment.subprocess, + "run", + lambda command, **kwargs: calls.append((command, kwargs)), + ) + + prepare_environment._install_headless_opencv() + + assert calls[0][0][-2:] == ["--yes", "opencv-python"] + assert calls[0][1]["check"] is False + assert calls[1][0][-1] == "opencv-python-headless==4.11.0.86" + assert calls[1][1]["check"] is True + + def test_manifest_owns_the_exact_camera_control_workload() -> None: _, manifest, case = e2e.CASES["sana-wm-bidirectional"] assert manifest["video_num_frames"] == 321 @@ -88,6 +267,7 @@ def test_raw_snapshot_calls_declared_official_entrypoint(monkeypatch, tmp_path: (model_dir / "dit").mkdir(parents=True) (model_dir / "vae").mkdir() (model_dir / "refiner/text_encoder").mkdir(parents=True) + stage1_text_encoder = _write_stage1_text_encoder(model_dir) (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") assert not (model_dir / "model_index.json").exists() @@ -126,7 +306,9 @@ def decode(video_path: Path, frames_dir: Path): result = e2e._official_reference(model_dir, manifest, case, tmp_path) command = captured["command"] - assert command[1] == str(entrypoint) + assert command[1] == str(e2e.TEST_ROOT / "official_reference.py") + assert command[command.index("--reference-repo") + 1] == str(source) + assert command[command.index("--stage1-text-encoder") + 1] == str(stage1_text_encoder) assert command[command.index("--action") + 1] == case["action"] assert command[command.index("--intrinsics") + 1] == str( e2e._asset(case["camera_intrinsics_file"]) @@ -174,6 +356,7 @@ def test_official_reference_dependency_failure_is_not_hidden(monkeypatch, tmp_pa model_dir = tmp_path / "raw-checkpoint" (model_dir / "dit").mkdir(parents=True) (model_dir / "refiner/text_encoder").mkdir(parents=True) + _write_stage1_text_encoder(model_dir) (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") monkeypatch.setenv("TRTMC_REFERENCE_SOURCE_DIR", str(source)) @@ -230,6 +413,7 @@ def test_official_refiner_requires_every_output_frame( model_dir = tmp_path / "checkpoint" (model_dir / "dit").mkdir(parents=True) (model_dir / "refiner/text_encoder").mkdir(parents=True) + _write_stage1_text_encoder(model_dir) (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") monkeypatch.setenv("TRTMC_REFERENCE_SOURCE_DIR", str(source)) diff --git a/families/segformer/tests/benchmark/reference.py b/families/segformer/tests/benchmark/reference.py new file mode 100644 index 0000000000..f8f637bcfd --- /dev/null +++ b/families/segformer/tests/benchmark/reference.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers SegFormer reference for Accuracy qualification.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import Any, Mapping, Sequence + +from qualification_tests.benchmark_qualification.performance import reference_harness + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _dtype(torch: Any, precision: str) -> Any: + try: + return {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + + +def _accuracy(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForSemanticSegmentation + + model_id = str(request["model"]) + revision = request.get("revision") + options = {"revision": revision} if revision else {} + processor = AutoImageProcessor.from_pretrained(model_id, use_fast=False, **options) + model = ( + AutoModelForSemanticSegmentation.from_pretrained( + model_id, + torch_dtype=_dtype(torch, str(request.get("precision", "fp32"))), + **options, + ) + .eval() + .to("cuda") + ) + result = [] + for sample in samples: + image = Image.open(str(sample["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + with torch.inference_mode(): + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(image.height, image.width)] + )[0].cpu() + result.append( + { + "sample_id": str(sample["sample_id"]), + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + "mask": mask.reshape(-1).tolist(), + } + ) + arguments.output.write_text(json.dumps({"samples": result}, indent=2) + "\n", encoding="utf-8") + return 0 + + +def _performance_session( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> reference_harness.Session: + import torch + from PIL import Image + from transformers import AutoImageProcessor, AutoModelForSemanticSegmentation + + if arguments.mode != "hf-eager": + raise ValueError("SegFormer reference requires hf-eager mode") + options = {"revision": arguments.revision} if arguments.revision else {} + processor = AutoImageProcessor.from_pretrained(arguments.model, use_fast=False, **options) + model = ( + AutoModelForSemanticSegmentation.from_pretrained( + arguments.model, + torch_dtype=_dtype(torch, arguments.precision), + **options, + ) + .eval() + .to("cuda") + ) + image = Image.open(str(request["image_path"])).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + inputs = { + name: value.to( + device=model.device, + dtype=next(model.parameters()).dtype if value.is_floating_point() else value.dtype, + ) + for name, value in inputs.items() + } + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(image.height, image.width)] + )[0] + return { + "num_masks": 1, + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + } + + return reference_harness.Session(invoke, "transformers") + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(values) + return reference_harness.run(values, description=__doc__, load=_performance_session) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/segformer/tests/benchmark/segformer-b0-ade.yaml b/families/segformer/tests/benchmark/segformer-b0-ade.yaml new file mode 100644 index 0000000000..eb3b8d8382 --- /dev/null +++ b/families/segformer/tests/benchmark/segformer-b0-ade.yaml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: segformer-b0-ade +candidate: + family: segformer + checkpoint: nvidia/segformer-b0-finetuned-ade-512-512 + revision: 489d5cd81a0b59fab9b7ea758d3548ebe99677da + task: segmentation + precision: fp16 + bundle: segformer-b0-ade.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: imagenette-segmentation-parity + benchmark: imagenette_segmentation_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + gate: + min_pixel_accuracy: 0.99 + min_mean_iou: 0.94 + min_sample_pass_rate: 0.9 + +performance: + - name: segment + benchmark: semantic_segmentation_performance + operation: segment + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: segmentation-shape diff --git a/families/stablelm/tests/benchmark/stablelm2-1.6b.yaml b/families/stablelm/tests/benchmark/stablelm2-1.6b.yaml deleted file mode 100644 index 54149dfe34..0000000000 --- a/families/stablelm/tests/benchmark/stablelm2-1.6b.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -schema_version: trtmc.qualification/v1 -model: stablelm2-1.6b -candidate: - family: stablelm - checkpoint: stabilityai/stablelm-2-1_6b - revision: f499ead74c53749bd93cebc6ce8bc0d7bdf1eaef - task: text_generation - precision: fp16 - bundle: stablelm2-1.6b.bundle - build: - max_sequence_length: 384 - fp32_layers: [23] - -accuracy: - - name: mmlu-continuation - benchmark: mmlu_continuation - samples: 10 - prompt_token_limit: 192 - truncation_side: left - reference: - precision: fp16 - request: &greedy-generation - max_new_tokens: 64 - temperature: 0.0 - top_k: 1 - top_p: 1.0 - min_p: 0.0 - seed: -1 - repetition_penalty: 1.0 - use_chat_template: false - enable_thinking: false - do_sample: false - gate: - min_pass_rate: 0.9 - allowed_failures: 1 - -performance: - - name: generate-22 - benchmark: text_generation_performance - operation: generate - request: - <<: *greedy-generation - prompt: |- - The following are multiple choice questions (with answers) about abstract algebra. - - Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field. - A. 0 - B. 1 - C. 2 - D. 3 - Answer: B - - Statement 1 | If aH is an element of a factor group, then |aH| divides |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: B - - Statement 1 | Every element of a group generates a cyclic subgroup of the group. Statement 2 | The symmetric group S_10 has 10 elements. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: C - - Statement 1| Every function from a finite set onto itself must be one to one. Statement 2 | Every subgroup of an abelian group is abelian. - A. True, True - B. False, False - C. True, False - D. False, True - Answer: A - - Find the characteristic of the ring 2Z. - A. 0 - B. 3 - C. 12 - D. 30 - Answer: A - - Let p = (1, 2, 5, 4)(2, 3) in S_5 . Find the index of

in S_5. - A. 8 - B. 2 - C. 24 - D. 120 - Answer: - max_new_tokens: 22 - measurement: - warmup: 5 - iterations: 10 - reference: - runner: hf-transformers - mode: torch-compile - fallback: hf-eager - compile_scope: model.forward - precision: fp16 - output_contract: exact-token-ids diff --git a/families/starcoder2/tests/benchmark/starcoder2-3b.yaml b/families/starcoder2/tests/benchmark/starcoder2-3b.yaml new file mode 100644 index 0000000000..428dc7cbe6 --- /dev/null +++ b/families/starcoder2/tests/benchmark/starcoder2-3b.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: starcoder2-3b +candidate: + family: starcoder2 + checkpoint: bigcode/starcoder2-3b + revision: 733247c55e3f73af49ce8e9c7949bf14af205928 + task: text_generation + precision: fp16 + bundle: starcoder2-3b.bundle + build: + max_sequence_length: 257 + +accuracy: + - name: humaneval-continuation + benchmark: humaneval_continuation + samples: 10 + prompt_token_limit: 192 + truncation_side: left + reference: + precision: fp32 + request: &greedy-generation + max_new_tokens: 64 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-generation + prompt: "def f():\n 0.5" + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: exact-token-ids diff --git a/families/t5/tests/benchmark/t5-small.yaml b/families/t5/tests/benchmark/t5-small.yaml new file mode 100644 index 0000000000..98c7f60b8d --- /dev/null +++ b/families/t5/tests/benchmark/t5-small.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: t5-small +candidate: + family: t5 + checkpoint: google-t5/t5-small + revision: df1b051c49625cf57a3d0d8d3863ed4d13564fe4 + task: text_generation + precision: fp16 + bundle: t5-small.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: wmt14-en-de + benchmark: wmt14_en_de_translation + samples: 10 + prompt_token_limit: 96 + truncation_side: right + reference: + precision: fp32 + task: seq2seq-lm + output_token_policy: strip-start + request: &greedy-translation + max_new_tokens: 128 + temperature: 0.0 + top_k: 1 + top_p: 1.0 + min_p: 0.0 + seed: -1 + repetition_penalty: 1.0 + use_chat_template: false + enable_thinking: false + do_sample: false + gate: + min_pass_rate: 0.9 + allowed_failures: 1 + +performance: + - name: generate-20 + benchmark: text_generation_performance + operation: generate + request: + <<: *greedy-translation + prompt: "translate English to German: The house is wonderful." + max_new_tokens: 20 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + task: seq2seq-lm + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + output_token_policy: strip-start + precision: fp32 + output_contract: exact-token-ids diff --git a/families/timesfm/tests/benchmark/reference.py b/families/timesfm/tests/benchmark/reference.py new file mode 100644 index 0000000000..4e9af1b9a9 --- /dev/null +++ b/families/timesfm/tests/benchmark/reference.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TimesFM-owned ETTh1 Accuracy reference.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if request.get("precision") != "fp32": + raise ValueError("TimesFM Accuracy requires an fp32 reference") + if not torch.cuda.is_available(): + raise RuntimeError("TimesFM Accuracy reference requires CUDA") + options = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + model = ( + transformers.TimesFmModelForPrediction.from_pretrained( + str(request["model"]), torch_dtype=torch.float32, **options + ) + .eval() + .to("cuda") + ) + context = int(model.config.context_length) + samples = [] + with torch.inference_mode(): + for sample in request["samples"]: + raw = [float(value) for value in sample["past_values"]] + count = min(len(raw), context) + values = [0.0] * context + values[-count:] = raw[-count:] + padding = [1] * context + padding[-count:] = [0] * count + decoded = model.decoder( + past_values=torch.tensor( + values, dtype=torch.float32, device="cuda" + ).reshape(1, context), + past_values_padding=torch.tensor( + padding, dtype=torch.int32, device="cuda" + ).reshape(1, context), + freq=torch.tensor( + [[int(sample.get("frequency", 0))]], + dtype=torch.long, + device="cuda", + ), + output_attentions=False, + output_hidden_states=False, + ) + output = model._postprocess_output( + decoded.last_hidden_state, (decoded.loc, decoded.scale) + )[:, -1, : model.config.horizon_length, 0].detach().float().cpu() + samples.append( + { + "sample_id": str(sample["sample_id"]), + "shape": [int(size) for size in output.shape], + "values": output.reshape(-1).tolist(), + } + ) + arguments.output.write_text( + json.dumps( + { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": request.get("revision"), + "precision": "fp32", + "samples": samples, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml b/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml new file mode 100644 index 0000000000..d0e00fffa0 --- /dev/null +++ b/families/timesfm/tests/benchmark/timesfm-2.0-500m-official.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: timesfm-2.0-500m-official +candidate: + family: timesfm + checkpoint: google/timesfm-2.0-500m-pytorch + revision: dc2443792ce5516872b89b37cf1bc058c3bf0c10 + task: time_series_forecast + precision: fp32 + bundle: timesfm-2.0-500m-official.bundle + +accuracy: + - name: etth1-parity + benchmark: etth1_forecast_parity + samples: 10 + reference: + command: reference.py + precision: fp32 + window: + columns: [OT] + context_length: 2048 + prediction_length: 128 + stride: 24 + test_target_start: 11520 + test_end: 14400 + frequency: 0 + gate: + max_relative_l2: 4.0e-3 + max_absolute_error: 7.0e-3 + +performance: + - name: forecast-128 + benchmark: time_series_performance + operation: solve + request: + past_values: [0.05, 0.15, 0.3, 0.5, 0.65, 0.8, 0.95, 1.1, 1.18, 1.24, 1.28, 1.31] + frequency: 2 + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: pytorch-timeseries + adapter_options: {reference_type: timesfm} + mode: pytorch-eager + precision: fp32 + output_contract: forecast-parity + max_relative_l2: 4.0e-3 + max_absolute_error: 7.0e-3 diff --git a/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml b/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml new file mode 100644 index 0000000000..f41ce9a242 --- /dev/null +++ b/families/timm_convnext/tests/benchmark/convnext-tiny-in12k-ft-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: convnext-tiny-in12k-ft-in1k +candidate: + family: timm_convnext + checkpoint: timm/convnext_tiny.in12k_ft_in1k + revision: aa096f03029c7f0ec052013f64c819b34f8ad790 + task: image_to_class_scores + precision: fp16 + bundle: convnext-tiny-in12k-ft-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml b/families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml new file mode 100644 index 0000000000..0ec7fe66b8 --- /dev/null +++ b/families/timm_crossvit/tests/benchmark/crossvit-9-240-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: crossvit-9-240-in1k +candidate: + family: timm_crossvit + checkpoint: timm/crossvit_9_240.in1k + revision: df3051d20d8addd16d03c4230046128c17e32826 + task: classification + precision: fp16 + bundle: crossvit-9-240-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml new file mode 100644 index 0000000000..08a5f80211 --- /dev/null +++ b/families/timm_densenet/tests/benchmark/densenet121-ra-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: densenet121-ra-in1k +candidate: + family: timm_densenet + checkpoint: timm/densenet121.ra_in1k + revision: 92007b6200e0b4a4fe68cb4e3947022a928aaaae + task: image_to_class_scores + precision: fp16 + bundle: densenet121-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml b/families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml new file mode 100644 index 0000000000..05099a87ce --- /dev/null +++ b/families/timm_dpn/tests/benchmark/dpn68b-ra-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dpn68b-ra-in1k +candidate: + family: timm_dpn + checkpoint: timm/dpn68b.ra_in1k + revision: fb49a617159bd2a0d6099742a6c57ef2d884dad9 + task: classification + precision: fp16 + bundle: dpn68b-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml b/families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml new file mode 100644 index 0000000000..e1ca938ee8 --- /dev/null +++ b/families/timm_dpn/tests/benchmark/dpn92-mx-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dpn92-mx-in1k +candidate: + family: timm_dpn + checkpoint: timm/dpn92.mx_in1k + revision: 8f2a71d2525254e14b0830a97250d841d5c4cae8 + task: classification + precision: fp16 + bundle: dpn92-mx-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml new file mode 100644 index 0000000000..06514d194d --- /dev/null +++ b/families/timm_efficientnet/tests/benchmark/efficientnet-b0-ra-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: efficientnet-b0-ra-in1k +candidate: + family: timm_efficientnet + checkpoint: timm/efficientnet_b0.ra_in1k + revision: 1b5383e5f79cc0f7fc067e372f8f26a5fa73f26a + task: image_to_class_scores + precision: fp16 + bundle: efficientnet-b0-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml b/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml new file mode 100644 index 0000000000..309e23494b --- /dev/null +++ b/families/timm_ghostnet/tests/benchmark/ghostnet-100-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: ghostnet-100-in1k +candidate: + family: timm_ghostnet + checkpoint: timm/ghostnet_100.in1k + revision: e524c2ee5d5a9412f802ce38ff895aba07b30910 + task: image_to_class_scores + precision: fp16 + bundle: ghostnet-100-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml b/families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml new file mode 100644 index 0000000000..92af0f0ee0 --- /dev/null +++ b/families/timm_hrnet/tests/benchmark/hrnet-w18-ms-aug-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: hrnet-w18-ms-aug-in1k +candidate: + family: timm_hrnet + checkpoint: timm/hrnet_w18.ms_aug_in1k + revision: 7e2c5583769f54514fd87e3ba9de408e33eaba0f + task: classification + precision: fp16 + bundle: hrnet-w18-ms-aug-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml new file mode 100644 index 0000000000..8fc6b730d4 --- /dev/null +++ b/families/timm_inception/tests/benchmark/inception-v3-tv-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: inception-v3-tv-in1k +candidate: + family: timm_inception + checkpoint: timm/inception_v3.tv_in1k + revision: 393d84cc85c467d8fbc0dc81a65c04e87a32572c + task: image_to_class_scores + precision: fp16 + bundle: inception-v3-tv-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml b/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml new file mode 100644 index 0000000000..fd0584220e --- /dev/null +++ b/families/timm_inception_resnet/tests/benchmark/inception-resnet-v2-tf-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: inception-resnet-v2-tf-in1k +candidate: + family: timm_inception_resnet + checkpoint: timm/inception_resnet_v2.tf_in1k + revision: 836b07d7d599da247de92f148c7a07fda18afced + task: image_to_class_scores + precision: fp16 + bundle: inception-resnet-v2-tf-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml b/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml new file mode 100644 index 0000000000..2869f6a1d0 --- /dev/null +++ b/families/timm_inception_v4/tests/benchmark/inception-v4-tf-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: inception-v4-tf-in1k +candidate: + family: timm_inception_v4 + checkpoint: timm/inception_v4.tf_in1k + revision: ad5d294cda312745a9afa433e45d9ccd956a1548 + task: image_to_class_scores + precision: fp16 + bundle: inception-v4-tf-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml b/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml new file mode 100644 index 0000000000..830276980b --- /dev/null +++ b/families/timm_mnasnet/tests/benchmark/mnasnet-100-rmsp-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mnasnet-100-rmsp-in1k +candidate: + family: timm_mnasnet + checkpoint: timm/mnasnet_100.rmsp_in1k + revision: a30af72360c7a4871b0156cc11d7b767208273d7 + task: classification + precision: fp16 + bundle: mnasnet-100-rmsp-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml new file mode 100644 index 0000000000..82fe4490c8 --- /dev/null +++ b/families/timm_mobilenetv2/tests/benchmark/mobilenetv2-100-ra-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv2-100-ra-in1k +candidate: + family: timm_mobilenetv2 + checkpoint: timm/mobilenetv2_100.ra_in1k + revision: 5afa12513b048c79b9147a5d210e9bdc50035481 + task: image_to_class_scores + precision: fp16 + bundle: mobilenetv2-100-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml new file mode 100644 index 0000000000..674895349a --- /dev/null +++ b/families/timm_mobilenetv3/tests/benchmark/mobilenetv3-large-100-ra-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv3-large-100-ra-in1k +candidate: + family: timm_mobilenetv3 + checkpoint: timm/mobilenetv3_large_100.ra_in1k + revision: 96f46a1c52932f27492dff66c72378eb99b443a7 + task: image_to_class_scores + precision: fp16 + bundle: mobilenetv3-large-100-ra-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml new file mode 100644 index 0000000000..58187d26af --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-large.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-large +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_large.e500_r256_in1k + revision: 033de0bd74b5261f7339c5de242a2e4785808aa3 + task: image_to_class_scores + precision: fp16 + bundle: mobilenetv4-conv-large.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml new file mode 100644 index 0000000000..77426a40a9 --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-medium.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-medium +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_medium.e500_r256_in1k + revision: ad66898c045c1b5223ea3f2c0830b74cf2e75bac + task: image_to_class_scores + precision: fp16 + bundle: mobilenetv4-conv-medium.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml new file mode 100644 index 0000000000..0865e90685 --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small-050.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-small-050 +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_small_050.e3000_r224_in1k + revision: 0ca27fbda87631d17b5a3d348af8e6b150dd9282 + task: image_to_class_scores + precision: fp16 + bundle: mobilenetv4-conv-small-050.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml new file mode 100644 index 0000000000..78ac621073 --- /dev/null +++ b/families/timm_mobilenetv4/tests/benchmark/mobilenetv4-conv-small.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilenetv4-conv-small +candidate: + family: timm_mobilenetv4 + checkpoint: timm/mobilenetv4_conv_small.e2400_r224_in1k + revision: 331fb803779522b685cf942e15f914fb6741c1eb + task: image_to_class_scores + precision: fp16 + bundle: mobilenetv4-conv-small.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml b/families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml new file mode 100644 index 0000000000..ccc5de07f5 --- /dev/null +++ b/families/timm_mobilevit/tests/benchmark/mobilevit-s-cvnets-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilevit-s-cvnets-in1k +candidate: + family: timm_mobilevit + checkpoint: timm/mobilevit_s.cvnets_in1k + revision: b531fe564ef88515b1aa1743ad4c40452b9f80ee + task: classification + precision: fp16 + bundle: mobilevit-s-cvnets-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml b/families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml new file mode 100644 index 0000000000..e3366f3447 --- /dev/null +++ b/families/timm_mobilevit/tests/benchmark/mobilevit-xxs-cvnets-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: mobilevit-xxs-cvnets-in1k +candidate: + family: timm_mobilevit + checkpoint: timm/mobilevit_xxs.cvnets_in1k + revision: 2c62d71504c6fbba9bb322f728586c4a235e5fa4 + task: classification + precision: fp16 + bundle: mobilevit-xxs-cvnets-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml b/families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml new file mode 100644 index 0000000000..ade48de911 --- /dev/null +++ b/families/timm_nfnet/tests/benchmark/dm-nfnet-f0-dm-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dm-nfnet-f0-dm-in1k +candidate: + family: timm_nfnet + checkpoint: timm/dm_nfnet_f0.dm_in1k + revision: ca8e830961c2ebceffff7affff34acfe7a1da673 + task: classification + precision: fp16 + bundle: dm-nfnet-f0-dm-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml b/families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml new file mode 100644 index 0000000000..602e4776d4 --- /dev/null +++ b/families/timm_nfnet/tests/benchmark/dm-nfnet-f3-dm-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: dm-nfnet-f3-dm-in1k +candidate: + family: timm_nfnet + checkpoint: timm/dm_nfnet_f3.dm_in1k + revision: ca2451bf0778ae0665a2f13d1b2a7fb878c9ff27 + task: classification + precision: fp16 + bundle: dm-nfnet-f3-dm-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml b/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml new file mode 100644 index 0000000000..869dde68ac --- /dev/null +++ b/families/timm_regnet/tests/benchmark/regnety-040-ra3-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: regnety-040-ra3-in1k +candidate: + family: timm_regnet + checkpoint: timm/regnety_040.ra3_in1k + revision: a1875159c7b04f3b3189af804642eebba1e2c118 + task: image_to_class_scores + precision: fp16 + bundle: regnety-040-ra3-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml b/families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml new file mode 100644 index 0000000000..c211ea9b23 --- /dev/null +++ b/families/timm_repvgg/tests/benchmark/repvgg-a2-rvgg-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: repvgg-a2-rvgg-in1k +candidate: + family: timm_repvgg + checkpoint: timm/repvgg_a2.rvgg_in1k + revision: 87d4d383cb45031cb9fa2fc8ddca73fd6649240f + task: classification + precision: fp16 + bundle: repvgg-a2-rvgg-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml b/families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml new file mode 100644 index 0000000000..841441df62 --- /dev/null +++ b/families/timm_res2net/tests/benchmark/res2net50-26w-4s-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: res2net50-26w-4s-in1k +candidate: + family: timm_res2net + checkpoint: timm/res2net50_26w_4s.in1k + revision: a36466d0354c9d337dbc0af329e13cb1cd7137fe + task: classification + precision: fp16 + bundle: res2net50-26w-4s-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml b/families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml new file mode 100644 index 0000000000..4d4d1ddd29 --- /dev/null +++ b/families/timm_res2net/tests/benchmark/res2net50d-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: res2net50d-in1k +candidate: + family: timm_res2net + checkpoint: timm/res2net50d.in1k + revision: 4f4bce4a1b5c17eabe5dde59af578ff9e1199ffc + task: classification + precision: fp16 + bundle: res2net50d-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_res2net/tests/benchmark/res2next50-in1k.yaml b/families/timm_res2net/tests/benchmark/res2next50-in1k.yaml new file mode 100644 index 0000000000..a3d1f3777c --- /dev/null +++ b/families/timm_res2net/tests/benchmark/res2next50-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: res2next50-in1k +candidate: + family: timm_res2net + checkpoint: timm/res2next50.in1k + revision: a8fe0f89e842b6dfa74b96deed2254d4fb7bf625 + task: classification + precision: fp16 + bundle: res2next50-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml b/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml new file mode 100644 index 0000000000..0b23a76c68 --- /dev/null +++ b/families/timm_resnest/tests/benchmark/resnest50d-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: resnest50d-in1k +candidate: + family: timm_resnest + checkpoint: timm/resnest50d.in1k + revision: 2239b0d822126dee3bc7a5357514de35b8f42092 + task: image_to_class_scores + precision: fp16 + bundle: resnest50d-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml new file mode 100644 index 0000000000..daa9f67758 --- /dev/null +++ b/families/timm_resnet/tests/benchmark/resnet50-a1-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: resnet50-a1-in1k +candidate: + family: timm_resnet + checkpoint: timm/resnet50.a1_in1k + revision: 767268603ca0cb0bfe326fa87277f19c419566ef + task: image_to_class_scores + precision: fp16 + bundle: resnet50-a1-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml b/families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml new file mode 100644 index 0000000000..31d4ed3f8e --- /dev/null +++ b/families/timm_senet/tests/benchmark/senet154-gluon-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: senet154-gluon-in1k +candidate: + family: timm_senet + checkpoint: timm/senet154.gluon_in1k + revision: 86f6d239ed30acbe3aca0cbb5769da2621a4e02c + task: classification + precision: fp16 + bundle: senet154-gluon-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml b/families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml new file mode 100644 index 0000000000..6a3b0b1910 --- /dev/null +++ b/families/timm_seresnet/tests/benchmark/seresnet50-a1-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: seresnet50-a1-in1k +candidate: + family: timm_seresnet + checkpoint: timm/seresnet50.a1_in1k + revision: 23f3482ee9acc4f51a2668a7cdf255fbd9420417 + task: classification + precision: fp16 + bundle: seresnet50-a1-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml b/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml new file mode 100644 index 0000000000..d2cd79761b --- /dev/null +++ b/families/timm_swin/tests/benchmark/swin-tiny-patch4-window7-224-ms-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: swin-tiny-patch4-window7-224-ms-in1k +candidate: + family: timm_swin + checkpoint: timm/swin_tiny_patch4_window7_224.ms_in1k + revision: 4cc3a7275b50b53a7bec45f32c236ebe64227cff + task: image_to_class_scores + precision: fp16 + bundle: swin-tiny-patch4-window7-224-ms-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml b/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml new file mode 100644 index 0000000000..a9caf433e5 --- /dev/null +++ b/families/timm_vgg/tests/benchmark/vgg16-tv-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: vgg16-tv-in1k +candidate: + family: timm_vgg + checkpoint: timm/vgg16.tv_in1k + revision: b8d8aa2dd860af9233c8c67385a8097fd6c35d3f + task: classification + precision: fp16 + bundle: vgg16-tv-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml new file mode 100644 index 0000000000..ec37c0bc34 --- /dev/null +++ b/families/timm_vit/tests/benchmark/timm-vit-base-p16-224-augreg-in21k-ft-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: timm-vit-base-p16-224-augreg-in21k-ft-in1k +candidate: + family: timm_vit + checkpoint: timm/vit_base_patch16_224.augreg_in21k_ft_in1k + revision: 2ec9fb3d7bb664aac471ac44582c94d18de33780 + task: image_to_class_scores + precision: fp16 + bundle: timm-vit-base-p16-224-augreg-in21k-ft-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml b/families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml new file mode 100644 index 0000000000..03fb8c4f41 --- /dev/null +++ b/families/timm_xception/tests/benchmark/xception41-tf-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xception41-tf-in1k +candidate: + family: timm_xception + checkpoint: timm/xception41.tf_in1k + revision: 8a17189361e63c972815ef62f2a30dd5b9f393b1 + task: classification + precision: fp16 + bundle: xception41-tf-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml b/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml new file mode 100644 index 0000000000..beaa2ebe46 --- /dev/null +++ b/families/timm_xcit/tests/benchmark/xcit-nano-12-p16-224-fb-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xcit-nano-12-p16-224-fb-in1k +candidate: + family: timm_xcit + checkpoint: timm/xcit_nano_12_p16_224.fb_in1k + revision: b181b4623913d78fd0927601c36e40b9d8bc74ce + task: image_to_class_scores + precision: fp16 + bundle: xcit-nano-12-p16-224-fb-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml b/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml new file mode 100644 index 0000000000..2c5ed7c2f4 --- /dev/null +++ b/families/timm_xcit/tests/benchmark/xcit-tiny-12-p16-224-fb-in1k.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xcit-tiny-12-p16-224-fb-in1k +candidate: + family: timm_xcit + checkpoint: timm/xcit_tiny_12_p16_224.fb_in1k + revision: 060449891838eaf301bdb3673e3daaf58e0370de + task: image_to_class_scores + precision: fp16 + bundle: xcit-tiny-12-p16-224-fb-in1k.bundle + build: + max_sequence_length: 1 + +accuracy: + - name: imagenette-parity + benchmark: imagenette_classification + samples: 100 + reference: + precision: fp32 + batch_size: 16 + gate: + max_top1_accuracy_drop_from_hf: 0.01 + min_top1_agreement: 0.98 + +performance: + - name: classify + benchmark: image_classification_performance + operation: classify + request: + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: timm-classification + mode: hf-eager + reference_backend: timm + precision: fp32 + output_contract: classification-top-class diff --git a/families/wan2_2_ti2v/requirements.txt b/families/wan2_2_ti2v/requirements.txt index a5ad62b1ec..bed38f0736 100644 --- a/families/wan2_2_ti2v/requirements.txt +++ b/families/wan2_2_ti2v/requirements.txt @@ -8,7 +8,7 @@ decord2==3.4.0 diffusers>=0.31.0 easydict einops==0.8.1 -flash-attn +flash-attn==2.8.3 ftfy==6.3.1 imageio[ffmpeg] librosa diff --git a/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml b/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml new file mode 100644 index 0000000000..e17ac3f375 --- /dev/null +++ b/families/wan2_2_ti2v/tests/benchmark/wan22-ti2v-5b.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: wan22-ti2v-5b +candidate: {family: wan2_2_ti2v, checkpoint: Wan-AI/Wan2.2-TI2V-5B, revision: 921dbaf3f1674a56f47e83fb80a34bac8a8f203e, task: image_generation, precision: bf16, bundle: wan22-ti2v-5b.bundle, build: {image_height: 704, image_width: 1280, video_num_frames: 121}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage, media_type: video, seed: 42, num_steps: 50, guidance_scale: 5.0, height: 704, width: 1280, video_num_frames: 121} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity, adapter_options: {model_id: Wan-AI/Wan2.2-TI2V-5B-Diffusers, model_revision: b8fff7315c768468a5333511427288870b2e9635, vae_class: AutoencoderKLWan, vae_precision: fp32}} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/wan2_2_ti2v/tests/test_reference.py b/families/wan2_2_ti2v/tests/test_reference.py index 262acc177f..fdcbcea258 100644 --- a/families/wan2_2_ti2v/tests/test_reference.py +++ b/families/wan2_2_ti2v/tests/test_reference.py @@ -144,7 +144,7 @@ def test_official_reference_declares_its_real_import_dependencies() -> None: "dashscope", "diffusers>=0.31.0", "easydict", - "flash-attn", + "flash-attn==2.8.3", "imageio[ffmpeg]", "librosa", "opencv-python>=4.9.0.80", diff --git a/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml b/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml new file mode 100644 index 0000000000..ab99b990fb --- /dev/null +++ b/families/wan_t2v/tests/benchmark/wan21-t2v-1.3b.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: wan21-t2v-1.3b +candidate: {family: wan_t2v, checkpoint: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, revision: 0fad780a534b6463e45facd96134c9f345acfa5b, task: image_generation, precision: fp16, bundle: wan21-t2v-1.3b.bundle, build: {context_parallel_size: 1, fp32_layers: [24], image_height: 480, image_width: 832, video_num_frames: 17}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A cat sitting on a beach watching the sunset, media_type: video, seed: 42, num_steps: 30, height: 480, width: 832, video_num_frames: 17} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: fp32, output_contract: media-artifact-parity} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-video, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/families/whisper/requirements.txt b/families/whisper/requirements.txt index 860aa73c19..be9ba894f6 100644 --- a/families/whisper/requirements.txt +++ b/families/whisper/requirements.txt @@ -1,4 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -scipy +scipy==1.15.3 +soundfile==0.14.0 +transformers==5.2.0 diff --git a/families/whisper/tests/benchmark/reference.py b/families/whisper/tests/benchmark/reference.py new file mode 100644 index 0000000000..27794eacd6 --- /dev/null +++ b/families/whisper/tests/benchmark/reference.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Transformers Whisper reference for Accuracy qualification.""" + +from __future__ import annotations + +from typing import Sequence + +from qualification_tests.benchmark_qualification.references import speech_io + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = speech_io.parser().parse_args(argv) + request = speech_io.load_request(arguments.request) + samples = speech_io.prepare_samples(request, arguments.output) + + import torch + from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + + model_id, revision = speech_io.model_id(request) + options = {"revision": revision} if revision else {} + processor = AutoProcessor.from_pretrained(model_id, **options) + model = AutoModelForSpeechSeq2Seq.from_pretrained( + model_id, + torch_dtype=speech_io.torch_dtype(torch, request), + **options, + ).eval().to("cuda") + max_new_tokens = int(request.get("max_new_tokens", 128)) + language = request.get("language") + texts = [] + for sample in samples: + inputs = processor( + sample.waveform, + sampling_rate=sample.sample_rate, + return_tensors="pt", + ) + model_dtype = next(model.parameters()).dtype + inputs = { + key: ( + value.to(device=model.device, dtype=model_dtype) + if value.is_floating_point() + else value.to(device=model.device) + ) + for key, value in inputs.items() + } + generation = {"max_new_tokens": max_new_tokens} + if isinstance(language, str) and language: + generation["language"] = language + with torch.inference_mode(): + token_ids = model.generate(**inputs, **generation) + texts.append(processor.batch_decode(token_ids, skip_special_tokens=True)[0]) + speech_io.write_result(arguments.output, samples, texts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml b/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml new file mode 100644 index 0000000000..9caf9e4c79 --- /dev/null +++ b/families/whisper/tests/benchmark/whisper-large-v3-turbo.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: whisper-large-v3-turbo +candidate: + family: whisper + checkpoint: openai/whisper-large-v3-turbo + revision: 41f01f3fe87f28c78e2fbf8b568835947dd65ed9 + task: transcription + precision: fp32 + bundle: whisper-large-v3-turbo.bundle + build: + max_sequence_length: 128 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 120 + language: en + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/whisper/tests/benchmark/whisper-small-fp16.yaml b/families/whisper/tests/benchmark/whisper-small-fp16.yaml new file mode 100644 index 0000000000..3777e23685 --- /dev/null +++ b/families/whisper/tests/benchmark/whisper-small-fp16.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: whisper-small-fp16 +candidate: + family: whisper + checkpoint: openai/whisper-small + revision: 973afd24965f72e36ca33b3055d56a652f456b4d + task: transcription + precision: fp16 + bundle: whisper-small-fp16.bundle + build: + max_sequence_length: 128 + fp32_layers: [0] + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 120 + language: en + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/Recording.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml b/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml new file mode 100644 index 0000000000..4900ee70bf --- /dev/null +++ b/families/whisper/tests/benchmark/whisper-tiny-fp16.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: whisper-tiny-fp16 +candidate: + family: whisper + checkpoint: openai/whisper-tiny + revision: 169d4a4341b33bc18d8881c4b69c2e104e1cc0af + task: transcription + precision: fp16 + bundle: whisper-tiny-fp16.bundle + build: + max_sequence_length: 128 + fp32_layers: [0] + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: librispeech-wer + benchmark: librispeech_transcription + samples: 10 + request: &request + max_new_tokens: 120 + language: en + reference: + command: reference.py + precision: fp32 + gate: + max_wer_to_reference: 0.1 + max_wer_increase_from_reference: 0.02 + +performance: + - name: transcribe + benchmark: speech_transformer_performance + operation: transcribe + request: + <<: *request + audio_path: ../data/librispeech-test-clean-6930-75918-0003.wav + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + adapter: hf-transformers-asr + mode: hf-eager + reference_backend: hf_transformers + precision: fp32 + output_contract: transcription-text diff --git a/families/olmo/tests/benchmark/olmo-1b.yaml b/families/xglm/tests/benchmark/xglm-564m.yaml similarity index 89% rename from families/olmo/tests/benchmark/olmo-1b.yaml rename to families/xglm/tests/benchmark/xglm-564m.yaml index d33e4f7315..3bb2f771ba 100644 --- a/families/olmo/tests/benchmark/olmo-1b.yaml +++ b/families/xglm/tests/benchmark/xglm-564m.yaml @@ -2,13 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 schema_version: trtmc.qualification/v1 -model: olmo-1b +model: xglm-564m candidate: - family: olmo - checkpoint: allenai/OLMo-1B-hf + family: xglm + checkpoint: facebook/xglm-564M + revision: f3059f01b98ccc877c673149e0178c0e957660f9 task: text_generation precision: fp16 - bundle: olmo-1b.bundle + bundle: xglm-564m.bundle build: max_sequence_length: 256 diff --git a/families/xlnet/tests/benchmark/xlnet-base.yaml b/families/xlnet/tests/benchmark/xlnet-base.yaml new file mode 100644 index 0000000000..887f702194 --- /dev/null +++ b/families/xlnet/tests/benchmark/xlnet-base.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: xlnet-base +candidate: + family: xlnet + checkpoint: xlnet/xlnet-base-cased + revision: ceaa69c7bc5e512b5007106a7ccbb7daf24b2c79 + task: encoding + precision: fp16 + bundle: xlnet-base.bundle + build: + max_sequence_length: 256 + +accuracy: + - name: stsbenchmark-parity + benchmark: stsbenchmark_embedding_parity + samples: 50 + reference: + precision: fp32 + gate: + min_vector_cosine: 0.999 + min_vector_pass_rate: 1.0 + max_pair_cosine_abs_delta: 0.02 + +performance: + - name: encode + benchmark: encoder_performance + operation: encode + request: + prompt: The capital of France is Paris. + measurement: + warmup: 5 + iterations: 10 + reference: + runner: hf-transformers + mode: torch-compile + fallback: hf-eager + compile_scope: model.forward + precision: fp32 + output_contract: embedding-shape diff --git a/families/yolo11/requirements.txt b/families/yolo11/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolo11/requirements.txt +++ b/families/yolo11/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolo11/tests/benchmark/prepare_environment.py b/families/yolo11/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolo11/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolo11/tests/benchmark/reference.py b/families/yolo11/tests/benchmark/reference.py new file mode 100644 index 0000000000..2647d54d08 --- /dev/null +++ b/families/yolo11/tests/benchmark/reference.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLO11 archive for qualification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +ARCHIVE = "yolo11n.pt" + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +def _load_model(model_dir: Path, precision: str) -> Any: + import torch + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + archive = model_dir / ARCHIVE + if not archive.is_file(): + raise FileNotFoundError(f"YOLO11 model directory has no {ARCHIVE}: {model_dir}") + blob = torch.load(str(archive), map_location="cpu", weights_only=False) + model = blob.get("model") if isinstance(blob, dict) else None + if model is None: + raise ValueError(f"YOLO11 archive has no model entry: {archive}") + return model.eval().to(device="cuda", dtype=dtype) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Any, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + from ultralytics.utils.ops import non_max_suppression + + with torch.inference_mode(): + raw = model(pixels) + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + kept = non_max_suppression( + raw, + conf_thres=threshold, + iou_thres=0.7, + max_det=300, + )[0] + rows = kept.detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in rows + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(rows), + "boxes": boxes, + "scores": [float(row[4]) for row in rows], + "class_ids": [int(row[5]) for row in rows], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), next(model.parameters()).dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolo11" or arguments.operation != "detect": + raise ValueError("YOLO11 reference only supports yolo11 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLO11 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), next(model.parameters()).dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolo11/tests/benchmark/yolo11n.yaml b/families/yolo11/tests/benchmark/yolo11n.yaml new file mode 100644 index 0000000000..359f9cc94b --- /dev/null +++ b/families/yolo11/tests/benchmark/yolo11n.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolo11n +candidate: + family: yolo11 + checkpoint: Ultralytics/YOLO11 + revision: 8b8ac7d1fae7468f85dbf89670dd66f41f485aab + task: object_detection + precision: fp16 + bundle: yolo11n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov10/requirements.txt b/families/yolov10/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolov10/requirements.txt +++ b/families/yolov10/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolov10/tests/benchmark/prepare_environment.py b/families/yolov10/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolov10/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolov10/tests/benchmark/reference.py b/families/yolov10/tests/benchmark/reference.py new file mode 100644 index 0000000000..5a92d44d6b --- /dev/null +++ b/families/yolov10/tests/benchmark/reference.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLOv10 archive for qualification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +def _load_model(model_dir: Path, precision: str) -> Any: + import torch + from ultralytics import YOLO + + from families.yolov10.checkpoint import Checkpoint + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + architecture = config.get("model") if isinstance(config, dict) else None + supported = {"yolov10n.yaml", "yolov10s.yaml", "yolov10x.yaml"} + if not isinstance(architecture, str) or architecture not in supported: + raise ValueError(f"unsupported YOLOv10 reference architecture: {architecture!r}") + model = YOLO(architecture, task="detect").model + checkpoint = Checkpoint.open(model_dir, framework="pt") + raw = {name: reader.get_tensor(name) for name, reader in checkpoint.tensor_map.items()} + state = { + name[len("model.") :]: value for name, value in raw.items() if name.startswith("model.") + } + missing, _ = model.load_state_dict(state, strict=False) + if missing: + raise ValueError(f"YOLOv10 reference is missing checkpoint tensors: {missing[:5]}") + return model.eval().to(device="cuda", dtype=dtype) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Any, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + + with torch.inference_mode(): + raw = model(pixels) + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + rows = raw[0] + kept = rows[rows[:, 4] >= threshold].detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in kept + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(kept), + "boxes": boxes, + "scores": [float(row[4]) for row in kept], + "class_ids": [int(row[5]) for row in kept], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), next(model.parameters()).dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolov10" or arguments.operation != "detect": + raise ValueError("YOLOv10 reference only supports yolov10 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLOv10 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), next(model.parameters()).dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolov10/tests/benchmark/yolov10n.yaml b/families/yolov10/tests/benchmark/yolov10n.yaml new file mode 100644 index 0000000000..8a5fc4bdc5 --- /dev/null +++ b/families/yolov10/tests/benchmark/yolov10n.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov10n +candidate: + family: yolov10 + checkpoint: jameslahm/yolov10n + revision: 3b91f578c5b492080f191a26aa35711bb44f608c + task: object_detection + precision: fp16 + bundle: yolov10n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov10/tests/benchmark/yolov10s.yaml b/families/yolov10/tests/benchmark/yolov10s.yaml new file mode 100644 index 0000000000..2855a485b4 --- /dev/null +++ b/families/yolov10/tests/benchmark/yolov10s.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov10s +candidate: + family: yolov10 + checkpoint: jameslahm/yolov10s + revision: 9096a248f4ab5dd21b3d9e9fa0295534db1b87ff + task: object_detection + precision: fp16 + bundle: yolov10s.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov10/tests/benchmark/yolov10x.yaml b/families/yolov10/tests/benchmark/yolov10x.yaml new file mode 100644 index 0000000000..bd7b843435 --- /dev/null +++ b/families/yolov10/tests/benchmark/yolov10x.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov10x +candidate: + family: yolov10 + checkpoint: jameslahm/yolov10x + revision: 6d7c9b6871ffa50dfd2764fa896bc0e33294233c + task: object_detection + precision: fp16 + bundle: yolov10x.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov5/requirements.txt b/families/yolov5/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolov5/requirements.txt +++ b/families/yolov5/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolov5/tests/benchmark/prepare_environment.py b/families/yolov5/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolov5/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolov5/tests/benchmark/reference.py b/families/yolov5/tests/benchmark/reference.py new file mode 100644 index 0000000000..0456198aa1 --- /dev/null +++ b/families/yolov5/tests/benchmark/reference.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLOv5 archive for qualification.""" + +from __future__ import annotations + +import argparse +import copy +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +@dataclass(frozen=True) +class Reference: + stages: Any + anchors: Any + strides: Any + head: tuple[Any, ...] + biases: tuple[Any, ...] + per_anchor: int + dtype: Any + + +def _load_model(model_dir: Path, precision: str) -> Reference: + import torch + from ultralytics.nn.tasks import parse_model + + from families.yolov5.checkpoint import _collect, _install_placeholders + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + archive_path = model_dir / "yolov5n.pt" + if not archive_path.is_file(): + raise FileNotFoundError(f"YOLOv5 model directory has no yolov5n.pt: {model_dir}") + _install_placeholders() + blob = torch.load(str(archive_path), map_location="cpu", weights_only=False) + archive = blob.get("model") if isinstance(blob, dict) else None + if archive is None: + raise ValueError(f"YOLOv5 archive has no model entry: {archive_path}") + spec = copy.deepcopy(archive.__dict__["yaml"]) + spec["head"].pop() + stages, _ = parse_model(spec, ch=3, verbose=False) + weights: dict[str, Any] = {} + _collect(archive, "", weights) + body = { + name[len("model.") :]: value.float() + for name, value in weights.items() + if name.startswith("model.") + } + head_index = str(len(stages)) + missing, unexpected = stages.load_state_dict( + {name: value for name, value in body.items() if not name.startswith(f"{head_index}.")}, + strict=False, + ) + if missing: + raise ValueError(f"YOLOv5 reference is missing checkpoint tensors: {missing[:5]}") + if any(not name.endswith("num_batches_tracked") for name in unexpected): + raise ValueError(f"YOLOv5 reference has unexpected checkpoint tensors: {unexpected[:5]}") + for module in stages.modules(): + if isinstance(module, torch.nn.BatchNorm2d): + module.eps = 1e-3 + classes = len(archive.__dict__["names"]) + strides = archive.__dict__["stride"].float().to(device="cuda", dtype=dtype) + anchors = weights[f"model.{head_index}.anchors"].float().to(device="cuda", dtype=dtype) + head = tuple( + body[f"{head_index}.m.{level}.weight"].to(device="cuda", dtype=dtype) + for level in range(len(strides)) + ) + biases = tuple( + body[f"{head_index}.m.{level}.bias"].to(device="cuda", dtype=dtype) + for level in range(len(strides)) + ) + return Reference( + stages=stages.eval().to(device="cuda", dtype=dtype), + anchors=anchors, + strides=strides, + head=head, + biases=biases, + per_anchor=classes + 5, + dtype=dtype, + ) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Reference, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + import torchvision + + with torch.inference_mode(): + outputs: list[Any] = [] + tensor = pixels + for stage in model.stages: + if stage.f != -1: + taken = [tensor if index == -1 else outputs[index] for index in stage.f] + tensor = stage(taken) + else: + tensor = stage(tensor) + outputs.append(tensor) + + levels = [] + for level, source_index in enumerate((17, 20, 23)): + raw = torch.nn.functional.conv2d( + outputs[source_index], + model.head[level], + model.biases[level], + ) + rows, columns = raw.shape[-2], raw.shape[-1] + values = raw.view(1, len(model.anchors[level]), model.per_anchor, rows, columns) + values = values.permute(0, 1, 3, 4, 2).sigmoid() + grid_y, grid_x = torch.meshgrid( + torch.arange(rows, device=raw.device), + torch.arange(columns, device=raw.device), + indexing="ij", + ) + grid = torch.stack((grid_x, grid_y), dim=-1).to(dtype=model.dtype) + centre = (values[..., 0:2] * 2 - 0.5 + grid) * model.strides[level] + extent = ( + (values[..., 2:4] * 2) ** 2 + * model.anchors[level].view(1, -1, 1, 1, 2) + * model.strides[level] + ) + levels.append( + torch.cat((centre, extent, values[..., 4:]), dim=-1).reshape( + 1, -1, model.per_anchor + ) + ) + predictions = torch.cat(levels, dim=1)[0] + confidence = predictions[:, 4:5] * predictions[:, 5:] + best, labels = confidence.max(dim=1) + survivors = best > threshold + centre = predictions[survivors, 0:2] + extent = predictions[survivors, 2:4] + corners = torch.cat((centre - extent / 2, centre + extent / 2), dim=1) + order = torchvision.ops.batched_nms( + corners, + best[survivors], + labels[survivors], + 0.7, + )[:300] + kept = torch.cat( + ( + corners[order], + best[survivors][order, None], + labels[survivors][order, None].to(dtype=model.dtype), + ), + dim=1, + ) + values = kept.detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in values + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(values), + "boxes": boxes, + "scores": [float(row[4]) for row in values], + "class_ids": [int(row[5]) for row in values], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), model.dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolov5" or arguments.operation != "detect": + raise ValueError("YOLOv5 reference only supports yolov5 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLOv5 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), model.dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolov5/tests/benchmark/yolov5n.yaml b/families/yolov5/tests/benchmark/yolov5n.yaml new file mode 100644 index 0000000000..4e3906de0d --- /dev/null +++ b/families/yolov5/tests/benchmark/yolov5n.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov5n +candidate: + family: yolov5 + checkpoint: Ultralytics/YOLOv5 + revision: 5bca797074771ecdfd6267d6e9be32ee201d937b + task: object_detection + precision: fp16 + bundle: yolov5n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/yolov8/requirements.txt b/families/yolov8/requirements.txt index 000a6d6dd8..ef46b844b1 100644 --- a/families/yolov8/requirements.txt +++ b/families/yolov8/requirements.txt @@ -2,4 +2,5 @@ # SPDX-License-Identifier: Apache-2.0 Pillow +opencv-python-headless==4.10.0.84 ultralytics==8.3.0 diff --git a/families/yolov8/tests/benchmark/prepare_environment.py b/families/yolov8/tests/benchmark/prepare_environment.py new file mode 100644 index 0000000000..4ae804ad2f --- /dev/null +++ b/families/yolov8/tests/benchmark/prepare_environment.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep the Ultralytics family environment independent of system OpenCV.""" + +from importlib import metadata +import subprocess +import sys + + +def main() -> None: + subprocess.run( + [sys.executable, "-m", "pip", "uninstall", "--yes", "opencv-python"], + check=True, + ) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--force-reinstall", + "--no-deps", + "opencv-python-headless==4.10.0.84", + ], + check=True, + ) + if metadata.version("opencv-python-headless") != "4.10.0.84": + raise RuntimeError("the headless OpenCV environment was not prepared") + + +if __name__ == "__main__": + main() diff --git a/families/yolov8/tests/benchmark/reference.py b/families/yolov8/tests/benchmark/reference.py new file mode 100644 index 0000000000..b84140ffa2 --- /dev/null +++ b/families/yolov8/tests/benchmark/reference.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the official Ultralytics YOLOv8 archive for qualification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Mapping, Sequence + + +ARCHIVE = "yolov8n.pt" + + +def _accuracy_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _performance_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--family", required=True) + parser.add_argument("--operation", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--adapter-options-json", required=True) + parser.add_argument("--timing-contract-json", required=True) + parser.add_argument("--padding", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--request-json", required=True) + parser.add_argument("--precision", required=True) + parser.add_argument("--mode", required=True) + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--selected-task", required=True) + parser.add_argument("--revision") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _offline_requested() -> bool: + return ( + os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + or os.environ.get("HF_HUB_OFFLINE") == "1" + ) + + +def _model_directory(model: str, revision: str | None, local_files_only: bool) -> Path: + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=model, + revision=revision, + local_files_only=local_files_only, + ) + ) + + +def _load_model(model_dir: Path, precision: str) -> Any: + import torch + + try: + dtype = {"fp16": torch.float16, "fp32": torch.float32}[precision] + except KeyError as error: + raise ValueError(f"unsupported reference precision {precision!r}") from error + archive = model_dir / ARCHIVE + if not archive.is_file(): + raise FileNotFoundError(f"YOLOv8 model directory has no {ARCHIVE}: {model_dir}") + blob = torch.load(str(archive), map_location="cpu", weights_only=False) + model = blob.get("model") if isinstance(blob, dict) else None + if model is None: + raise ValueError(f"YOLOv8 archive has no model entry: {archive}") + return model.eval().to(device="cuda", dtype=dtype) + + +def _prepare_image(image_path: str, dtype: Any) -> tuple[Any, dict[str, Any]]: + import numpy as np + from PIL import Image + import torch + + size = 640 + source = Image.open(image_path).convert("RGB") + scale = min(size / source.height, size / source.width) + resized = source.resize( + (round(source.width * scale), round(source.height * scale)), + Image.Resampling.BILINEAR, + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + pad_x = (size - resized.width) // 2 + pad_y = (size - resized.height) // 2 + canvas.paste(resized, (pad_x, pad_y)) + pixels = np.asarray(canvas, dtype=np.float32).transpose(2, 0, 1)[None].copy() / 255.0 + tensor = torch.from_numpy(pixels).to(device="cuda", dtype=dtype) + geometry = { + "height": source.height, + "width": source.width, + "scale": scale, + "pad_x": pad_x, + "pad_y": pad_y, + } + return tensor, geometry + + +def _detect( + model: Any, pixels: Any, geometry: Mapping[str, Any], threshold: float +) -> dict[str, Any]: + import torch + from ultralytics.utils.ops import non_max_suppression + + with torch.inference_mode(): + raw = model(pixels) + raw = raw[0] if isinstance(raw, (list, tuple)) else raw + kept = non_max_suppression( + raw, + conf_thres=threshold, + iou_thres=0.7, + max_det=300, + )[0] + rows = kept.detach().float().cpu().tolist() + scale = float(geometry["scale"]) + pad_x = int(geometry["pad_x"]) + pad_y = int(geometry["pad_y"]) + boxes = [ + [ + (float(row[0]) - pad_x) / scale, + (float(row[1]) - pad_y) / scale, + (float(row[2]) - pad_x) / scale, + (float(row[3]) - pad_y) / scale, + ] + for row in rows + ] + return { + "image_height": int(geometry["height"]), + "image_width": int(geometry["width"]), + "detections": len(rows), + "boxes": boxes, + "scores": [float(row[4]) for row in rows], + "class_ids": [int(row[5]) for row in rows], + } + + +def _accuracy(arguments: argparse.Namespace) -> int: + request = json.loads(arguments.request.read_text(encoding="utf-8")) + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("reference request must contain samples") + model_dir = _model_directory( + str(request["model"]), + request.get("revision"), + _offline_requested(), + ) + model = _load_model(model_dir, str(request.get("precision", "fp32"))) + threshold = float(request.get("request", {}).get("score_threshold", 0.25)) + results = [] + for sample in samples: + pixels, geometry = _prepare_image(str(sample["image_path"]), next(model.parameters()).dtype) + results.append( + {"sample_id": str(sample["sample_id"]), **_detect(model, pixels, geometry, threshold)} + ) + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": results}, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +def _performance(arguments: argparse.Namespace) -> int: + import torch + + if arguments.family != "yolov8" or arguments.operation != "detect": + raise ValueError("YOLOv8 reference only supports yolov8 detect") + if arguments.mode != "pytorch-eager": + raise ValueError("YOLOv8 reference only supports pytorch-eager") + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + request = _json_object(arguments.request_json, "--request-json") + timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + expected_timing = { + "timing_scope": "task-model-call-wall", + "input_preparation_included": False, + "asset_loading_included": False, + } + if timing != expected_timing: + raise ValueError(f"unsupported timing contract: {timing}") + model_dir = _model_directory( + arguments.model, + arguments.revision, + arguments.local_files_only, + ) + model = _load_model(model_dir, arguments.precision) + pixels, geometry = _prepare_image(str(request["image_path"]), next(model.parameters()).dtype) + threshold = float(request.get("score_threshold", 0.25)) + for _ in range(arguments.warmup): + _detect(model, pixels, geometry, threshold) + samples = [] + output_summary: dict[str, Any] = {} + for _ in range(arguments.iterations): + torch.cuda.synchronize() + started = time.perf_counter() + output_summary = _detect(model, pixels, geometry, threshold) + torch.cuda.synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + median = statistics.median(samples) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": "ultralytics", + "framework": "pytorch", + "model": arguments.model, + "family": arguments.family, + "operation": arguments.operation, + "case_name": arguments.case_name, + "selected_task": arguments.selected_task, + "precision": arguments.precision, + "mode": arguments.mode, + **expected_timing, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + **expected_timing, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": median, + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": { + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(0), + }, + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + values = list(sys.argv[1:] if argv is None else argv) + if "--request" in values: + return _accuracy(_accuracy_parser().parse_args(values)) + return _performance(_performance_parser().parse_args(values)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/families/yolov8/tests/benchmark/yolov8n.yaml b/families/yolov8/tests/benchmark/yolov8n.yaml new file mode 100644 index 0000000000..d7a1ed44ef --- /dev/null +++ b/families/yolov8/tests/benchmark/yolov8n.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schema_version: trtmc.qualification/v1 +model: yolov8n +candidate: + family: yolov8 + checkpoint: Ultralytics/YOLOv8 + revision: 27f858f2dc238503a66ce10248cf7ec71a3ded5d + task: object_detection + precision: fp16 + bundle: yolov8n.bundle + build: + max_sequence_length: 1 + +reference_environment: + requirements: ../../requirements.txt + +accuracy: + - name: coco2017-object-detection + benchmark: coco2017_object_detection + samples: 100 + label_space: coco-contiguous-80 + request: &detection-request + score_threshold: 0.25 + reference: + command: reference.py + precision: fp32 + gate: + max_map_50_95_drop: 0.02 + max_map_50_drop: 0.02 + +performance: + - name: detect + benchmark: object_detection_performance + operation: detect + request: + <<: *detection-request + image_path: ../data/test_img.jpeg + measurement: + warmup: 5 + iterations: 10 + reference: + runner: task-reference + script: tests/benchmark/reference.py + mode: pytorch-eager + reference_backend: ultralytics + precision: fp32 + output_contract: detection-parity + min_box_iou: 0.9 + max_score_abs_error: 0.005 diff --git a/families/z_image/tests/benchmark/z-image-turbo.yaml b/families/z_image/tests/benchmark/z-image-turbo.yaml new file mode 100644 index 0000000000..5cb95d6d60 --- /dev/null +++ b/families/z_image/tests/benchmark/z-image-turbo.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +schema_version: trtmc.qualification/v1 +model: z-image-turbo +candidate: {family: z_image, checkpoint: Tongyi-MAI/Z-Image-Turbo, revision: f332072aa78be7aecdf3ee76d5c247082da564a6, task: image_generation, precision: fp16, bundle: z-image-turbo.bundle, build: {fp32_layers: [2, 3, 4, 7, 8], image_height: 1024, image_width: 1024}} +reference_environment: {requirements: ../../requirements.txt} +accuracy: + - name: generated-media-parity + benchmark: task_output_parity + operation: generate_image + request: &request {prompt: A photo of a cat sitting on a windowsill at sunset, seed: 42, num_steps: 9, guidance_scale: 0.0, height: 1024, width: 1024} + reference: &reference {adapter: hf-diffusers, mode: hf-eager, precision: bf16, output_contract: media-artifact-parity} + gate: {min_psnr: 5.0, min_ssim: 0.1} +performance: + - {name: generate-image, benchmark: media_generation_performance, operation: generate_image, request: *request, measurement: {warmup: 5, iterations: 10}, reference: {<<: *reference, runner: task-reference, reference_backend: hf_diffusers, output_contract: media-shape}} diff --git a/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md b/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md index a5b971b443..f45783eb40 100644 --- a/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md +++ b/plugins/trtmc-agent-skills/skills/optimize-model-precision/SKILL.md @@ -45,7 +45,7 @@ trtmc-bench run --model --case \ --output ``` -Use `tools/perf_matrix.py` when the claim targets a checked-in release entry. +Use `python3 -m qualification_tests.benchmark_qualification.performance` when the claim targets a checked-in release entry. Report every attempt in a compact table, including effective request values, correctness, measured objective, confounders, and unrun targets. It is valid for the original configuration to remain the best qualified result. diff --git a/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md b/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md index e9fba4f0cd..b57ce684b9 100644 --- a/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md +++ b/plugins/trtmc-agent-skills/skills/profile-model/SKILL.md @@ -10,7 +10,7 @@ description: >- Choose the evidence level before running: - For one model or testcase, use `trtmc-bench`. -- For a checked-in release comparison, use `tools/perf_matrix.py`. +- For a checked-in release comparison, use `python3 -m qualification_tests.benchmark_qualification.performance`. The public benchmark boundary is `public_task_call_wall`. It excludes bundle building, process startup, warmup, report generation, and bundle loading. @@ -39,9 +39,9 @@ confounder instead of one causal speedup percentage. For release evidence, first check and then run the exact entry: ```bash -python3 tools/perf_matrix.py check \ +python3 -m qualification_tests.benchmark_qualification.performance check \ --environment --entry -python3 tools/perf_matrix.py run \ +python3 -m qualification_tests.benchmark_qualification.performance run \ --environment --entry ``` diff --git a/pyproject.toml b/pyproject.toml index cee00e7423..34f1d1ee47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,7 @@ include = [ "requirements", "third_party", "examples", - "tools/perf_matrix.py", + "qualification_tests/benchmark_qualification", ] exclude = [ "__pycache__", @@ -95,6 +95,7 @@ testpaths = [ "examples/models/cosmos3/dual_spark/test_cosmos3_dual_spark_source.py", "examples/models/nemotron_voicechat/full_duplex/test_voicechat_full_duplex_source.py", "tools/tests", + "qualification_tests/benchmark_qualification", ] markers = [ "gpu: tests requiring a CUDA GPU", diff --git a/tools/benchmark_qualification/__init__.py b/qualification_tests/__init__.py similarity index 65% rename from tools/benchmark_qualification/__init__.py rename to qualification_tests/__init__.py index 8ffcd736c3..826b2d60d4 100644 --- a/tools/benchmark_qualification/__init__.py +++ b/qualification_tests/__init__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Repository-only Accuracy and Performance qualification automation.""" +"""Repository-owned validation and qualification modules.""" diff --git a/qualification_tests/benchmark_qualification/__init__.py b/qualification_tests/benchmark_qualification/__init__.py new file mode 100644 index 0000000000..efc4c49e8a --- /dev/null +++ b/qualification_tests/benchmark_qualification/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Internal model qualification framework.""" diff --git a/qualification_tests/benchmark_qualification/accuracy.py b/qualification_tests/benchmark_qualification/accuracy.py new file mode 100644 index 0000000000..243dce20f0 --- /dev/null +++ b/qualification_tests/benchmark_qualification/accuracy.py @@ -0,0 +1,3366 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Internal Accuracy qualification through the installed user benchmark.""" + +from __future__ import annotations + +from array import array +from collections import defaultdict +import csv +from itertools import permutations +import json +import math +import os +import random +import re +from pathlib import Path +from typing import Any, Mapping, Sequence + +from trtmc_benchmark.artifact_metrics import ( + generated_audio_metrics, + generated_audio_passes, + generated_media_metrics, + generated_media_passes, + metric_geometry_metrics, + metric_geometry_passes, + robot_action_metrics, + robot_action_passes, +) +from qualification_tests.benchmark_qualification.performance import reference_protocol + +from .catalog import QualificationCase, QualificationError, load_benchmark +from .datasets import Dataset, resolve_dataset +from .runtime import ( + RuntimeContext, + prepare_bundle, + reference_environment_options, + reference_python, + require_candidate, + run_command, + write_model_descriptor, + write_result, +) + + +def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, Any]: + output = context.case_artifacts(case) + output.mkdir(parents=True, exist_ok=True) + definition = load_benchmark(context.repository, case) + metric = definition.get("metric") + metric_name = metric.get("name") if isinstance(metric, Mapping) else None + if metric_name == "task_output_parity": + result = _task_output_parity(case, context, output) + else: + dataset = resolve_dataset(definition, context, case.source) + if metric_name == "exact_token_ids": + result = _text_generation_parity(case, context, definition, dataset, output) + elif metric_name == "embedding_vector_parity": + result = _encoder_embedding_parity(case, context, dataset, output) + elif metric_name == "reranking_score_parity": + result = _reranking_score_parity(case, context, dataset, output) + elif metric_name == "forecast_tensor_parity": + result = _etth1(case, context, definition, dataset, output) + elif metric_name == "image_classification_top1_parity": + result = _image_classification_parity(case, context, dataset, output) + elif metric_name == "speech_transcription_wer_parity": + result = _speech_transcription_parity(case, context, definition, dataset, output) + elif metric_name == "image_feature_knn_parity": + result = _image_feature_knn_parity(case, context, dataset, output) + elif metric_name == "coco_object_detection_accuracy": + result = _coco_object_detection_accuracy(case, context, dataset, output) + elif metric_name == "prompted_segmentation_parity": + result = _prompted_segmentation_parity(case, context, dataset, output) + elif metric_name == "text_prompted_instance_segmentation_parity": + result = _text_prompted_instance_segmentation_parity(case, context, dataset, output) + elif metric_name == "semantic_segmentation_parity": + result = _semantic_segmentation_parity(case, context, dataset, output) + elif metric_name == "vision_language_text_parity": + result = _vision_language_text_parity(case, context, dataset, output) + elif metric_name == "ocr_text_parity": + result = _ocr_text_parity(case, context, definition, dataset, output) + elif metric_name == "localization_text_parity": + result = _localization_text_parity(case, context, dataset, output) + elif metric_name == "stereo_disparity_parity": + result = _stereo_disparity_parity(case, context, dataset, output) + elif metric_name == "metric_geometry_parity": + result = _metric_geometry_parity(case, context, dataset, output) + elif metric_name == "robot_action_parity": + result = _robot_action_parity(case, context, dataset, output) + elif metric_name == "task_output_parity": + pass + else: + raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") + write_result(output, result) + return result + + +def _task_output_parity( + case: QualificationCase, context: RuntimeContext, output: Path +) -> dict[str, Any]: + configured = case.values + request = configured.get("request", {}) + reference = configured.get("reference", {}) + gate = configured.get("gate", {}) + if not all(isinstance(value, Mapping) for value in (request, reference, gate)): + raise QualificationError("task-output request, reference, and gate must be objects") + request = _resolve_task_assets(case, request) + descriptor = write_model_descriptor(case, output, request, context=context) + descriptor_value = json.loads(descriptor.read_text(encoding="utf-8")) + adapter_value = reference.get("adapter") + script_value = reference.get("script") + family_root = case.source.parents[2].resolve() + if (adapter_value is None) == (script_value is None): + raise QualificationError("task-output reference requires exactly one adapter or script") + adapter = str(adapter_value) if adapter_value is not None else None + script = None + if script_value is not None: + relative = Path(str(script_value)) + script = (family_root / relative).resolve() + if ( + relative.is_absolute() + or ".." in relative.parts + or not script.is_relative_to(family_root) + or not script.is_file() + ): + raise QualificationError(f"task-output reference script is unavailable: {script_value}") + adapter_options = dict(reference.get("adapter_options", {})) + for name, value in tuple(adapter_options.items()): + if not name.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if not path.is_absolute(): + path = (case.source.parent / path).resolve() + if not path.is_relative_to(family_root) or not path.is_file(): + raise QualificationError(f"task-output reference asset {name!r} is unavailable: {path}") + adapter_options[name] = str(path) + adapter_options = reference_environment_options(case, context, adapter_options) + reference_path = output / "reference-result.json" + revision = reference.get("revision", descriptor_value.get("hf_revision")) + selected_task = case.candidate.get("selected_task") + command = reference_protocol.command( + python=reference_python(case, context), + generic_runner=( + context.repository + / "qualification_tests/benchmark_qualification/performance/references/generic_reference.py" + ), + script=script, + family=case.family, + operation=str(configured["operation"]), + manifest=descriptor, + selected_task=str(selected_task or case.candidate["task"]) if script else ( + str(selected_task) if selected_task else None + ), + testcase_name=case.name, + adapter=adapter, + adapter_options=adapter_options, + timing_contract={}, + padding="longest", + model=str(descriptor_value["hf_id"]), + revision=str(revision) if revision else None, + request=request, + precision=str(reference.get("precision", case.candidate["precision"])), + mode=str(reference.get("mode", "hf-eager")), + warmup=0, + iterations=1, + case_name=case.name, + output=reference_path, + trust_remote_code=bool(case.candidate.get("trust_remote_code", False)), + local_files_only=os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + ) + completed = run_command(command, output, "reference", timeout=7200, verbose=context.verbose) + if completed.returncode != 0: + raise QualificationError(f"task-output Accuracy reference failed; see {output}") + try: + reference_result = json.loads(reference_path.read_text(encoding="utf-8")) + expected = reference_result["output_summary"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as error: + raise QualificationError("task-output Accuracy reference returned no output") from error + actual_values, bundle = _candidate_outputs( + case, + context, + output, + str(configured["operation"]), + [{"sample_id": case.name, "request": request}], + ) + actual = actual_values[0] + contract = str(reference.get("output_contract", "")) + if not isinstance(expected, Mapping): + raise QualificationError("task-output Accuracy reference returned an invalid output") + try: + if contract == "media-artifact-parity": + metrics = generated_media_metrics(actual, expected) + passed = generated_media_passes(metrics, gate) + elif contract == "audio-artifact-parity": + metrics = generated_audio_metrics(actual, expected) + passed = generated_audio_passes(metrics, gate) + elif contract == "normalized-text": + candidate_text = _normalized_answer(actual.get("text")) + reference_text = _normalized_answer(expected.get("text")) + distance = _normalized_edit_distance(candidate_text, reference_text) + metrics = {"normalized_edit_distance": distance} + passed = bool(candidate_text) and distance <= float( + gate["max_normalized_edit_distance"] + ) + else: + raise QualificationError(f"unsupported task-output contract {contract!r}") + except (KeyError, TypeError, ValueError) as error: + raise QualificationError(f"task-output Accuracy comparison failed: {error}") from error + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "reference": { + "adapter": adapter, + "script": str(script) if script else None, + "model": reference_result.get("model"), + }, + "metrics": metrics, + "gate": dict(gate), + } + + +def _resolve_task_assets(case: QualificationCase, request: Mapping[str, Any]) -> dict[str, Any]: + resolved = dict(request) + family_root = case.source.parents[2].resolve() + for key, value in request.items(): + if not key.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if path.is_absolute(): + continue + path = (case.source.parent / path).resolve() + if not path.is_relative_to(family_root) or not path.is_file(): + raise QualificationError(f"task-output asset {key!r} is unavailable: {path}") + if key == "prompt_path": + payload = ( + json.loads(path.read_text(encoding="utf-8")) if path.suffix == ".json" else None + ) + prompt = ( + payload.get("prompt") + if isinstance(payload, Mapping) + else path.read_text(encoding="utf-8").strip() + ) + if not isinstance(prompt, str) or not prompt: + raise QualificationError(f"task-output prompt is unavailable: {path}") + resolved.pop(key, None) + resolved["prompt"] = prompt + else: + resolved[key] = str(path) + return resolved + + +def _robot_control_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("robot-control dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"robot-control dataset requires at least {sample_limit} requests") + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"robot-control request {index} must be an object") + sample = {"sample_id": str(request.get("id") or f"sample-{index}")} + for source, target in (("image", "image_path"), ("state", "state_path")): + relative = request.get(source) + if not isinstance(relative, str) or not relative: + raise QualificationError(f"robot-control request {index} has no {source}") + path = (root / relative).resolve() + if not path.is_relative_to(root) or not path.is_file(): + raise QualificationError( + f"robot-control request {index} {source} is unavailable: {path}" + ) + sample[target] = str(path) + selected.append(sample) + return selected + + +def _robot_action_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _robot_control_samples(dataset, sample_limit) + expected = _family_image_reference(case, context, output, selected) + actual, bundle = _candidate_outputs( + case, + context, + output, + "control", + [ + { + "sample_id": sample["sample_id"], + "request": { + "image_path": sample["image_path"], + "state_path": sample["state_path"], + }, + } + for sample in selected + ], + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("robot-action gate must be an object") + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("robot-action min_sample_pass_rate must be in [0, 1]") + thresholds = {key: value for key, value in gate.items() if key != "min_sample_pass_rate"} + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + try: + metrics = robot_action_metrics(candidate_sample, reference_sample) + passed = robot_action_passes(metrics, thresholds) + except ValueError as error: + raise QualificationError( + f"robot-action sample {sample['sample_id']} is invalid: {error}" + ) from error + rows.append({"sample_id": sample["sample_id"], "passed": passed, **metrics}) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": pass_rate, + **{name: max(float(row[name]) for row in rows) for name in thresholds}, + }, + "gate": dict(gate), + "samples": rows, + } + + +def _metric_geometry_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "metric-geometry") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("metric-geometry request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "geometry", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("metric-geometry gate must be an object") + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("metric-geometry min_sample_pass_rate must be in [0, 1]") + thresholds = {key: value for key, value in gate.items() if key != "min_sample_pass_rate"} + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + try: + metrics = metric_geometry_metrics(candidate_sample, reference_sample) + passed = metric_geometry_passes(metrics, thresholds) + except (OSError, ValueError) as error: + raise QualificationError( + f"metric-geometry sample {sample['sample_id']} is invalid: {error}" + ) from error + rows.append({"sample_id": sample["sample_id"], "passed": passed, **metrics}) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + metric_names = tuple(thresholds) + aggregate = { + name: ( + min(float(row[name]) for row in rows) + if name in {"mask_iou", "points_cosine"} + else max(float(row[name]) for row in rows) + ) + for name in metric_names + } + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": {"samples": len(rows), "sample_pass_rate": pass_rate, **aggregate}, + "gate": dict(gate), + "samples": rows, + } + + +def _stereo_samples(dataset: Dataset, sample_limit: int) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("stereo dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"stereo dataset requires at least {sample_limit} requests") + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"stereo request {index} must be an object") + sample = {"sample_id": str(request.get("id") or f"sample-{index}")} + for source, target in ( + ("left_image", "left_image_path"), + ("right_image", "right_image_path"), + ): + relative = request.get(source) + if not isinstance(relative, str) or not relative: + raise QualificationError(f"stereo request {index} has no {source}") + image = (root / relative).resolve() + if not image.is_relative_to(root) or not image.is_file(): + raise QualificationError(f"stereo request {index} {source} is unavailable: {image}") + sample[target] = str(image) + selected.append(sample) + return selected + + +def _disparity_values(value: Mapping[str, Any], label: str) -> list[float]: + height = value.get("height") + width = value.get("width") + count = value.get("element_count", value.get("disparity_pixels")) + artifact = value.get("disparity_artifact") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or isinstance(count, bool) + or not isinstance(count, int) + or count != height * width + or not isinstance(artifact, str) + ): + raise QualificationError(f"{label} disparity output is incomplete") + try: + payload = Path(artifact).read_bytes() + except OSError as error: + raise QualificationError(f"{label} disparity artifact is unavailable") from error + if len(payload) != count * 4: + raise QualificationError(f"{label} disparity artifact has the wrong size") + values = array("f") + values.frombytes(payload) + if len(values) != count: + raise QualificationError(f"{label} disparity artifact has the wrong element count") + return list(values) + + +def _stereo_disparity_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _stereo_samples(dataset, sample_limit) + request = configured.get("request", {}) + reference = configured.get("reference", {}) + if not isinstance(request, Mapping) or not isinstance(reference, Mapping): + raise QualificationError("stereo Accuracy request and reference must be objects") + command = reference.get("command") + if not isinstance(command, str) or not command: + raise QualificationError("stereo Accuracy reference.command must be set") + runner = (case.source.parent / command).resolve() + family_root = case.source.parents[2].resolve() + if not runner.is_relative_to(family_root) or not runner.is_file(): + raise QualificationError(f"stereo reference runner is not family-owned: {runner}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp16")), + "request": dict(request), + "samples": selected, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"stereo Accuracy reference failed; see {output}") + try: + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + except (OSError, json.JSONDecodeError) as error: + raise QualificationError("stereo Accuracy reference returned unreadable output") from error + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("stereo Accuracy reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + **dict(request), + "left_image_path": sample["left_image_path"], + "right_image_path": sample["right_image_path"], + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "disparity", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("stereo Accuracy gate must be an object") + minimum_cosine = float(gate.get("min_cosine", 0.999)) + maximum_mean_error = float(gate.get("max_mean_abs_error", 0.5)) + maximum_bad_fraction = float(gate.get("max_bad_2px_fraction", 0.03)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= minimum_cosine <= 1.0 or not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("stereo cosine and pass-rate gates must be in [0, 1]") + if ( + not math.isfinite(maximum_mean_error) + or maximum_mean_error < 0.0 + or not 0.0 <= maximum_bad_fraction <= 1.0 + ): + raise QualificationError("stereo error gates are invalid") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_values = _disparity_values(candidate_sample, "candidate") + reference_values = _disparity_values(reference_sample, "reference") + if len(candidate_values) != len(reference_values): + raise QualificationError("candidate and reference disparity shapes differ") + candidate_valid = all(math.isfinite(value) and value >= 0.0 for value in candidate_values) + reference_valid = all(math.isfinite(value) and value >= 0.0 for value in reference_values) + dot = math.fsum( + left * right for left, right in zip(candidate_values, reference_values, strict=True) + ) + candidate_norm = math.sqrt(math.fsum(value * value for value in candidate_values)) + reference_norm = math.sqrt(math.fsum(value * value for value in reference_values)) + cosine = ( + dot / (candidate_norm * reference_norm) if candidate_norm and reference_norm else 0.0 + ) + differences = [ + abs(left - right) + for left, right in zip(candidate_values, reference_values, strict=True) + ] + mean_error = math.fsum(differences) / len(differences) + bad_fraction = sum(value > 2.0 for value in differences) / len(differences) + passed = ( + candidate_valid + and reference_valid + and cosine >= minimum_cosine + and mean_error <= maximum_mean_error + and bad_fraction <= maximum_bad_fraction + ) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": passed, + "cosine": cosine, + "mean_abs_error": mean_error, + "bad_2px_fraction": bad_fraction, + } + ) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": pass_rate, + "min_cosine": min(row["cosine"] for row in rows), + "max_mean_abs_error": max(row["mean_abs_error"] for row in rows), + "max_bad_2px_fraction": max(row["bad_2px_fraction"] for row in rows), + }, + "gate": { + "min_cosine": minimum_cosine, + "max_mean_abs_error": maximum_mean_error, + "max_bad_2px_fraction": maximum_bad_fraction, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _text_generation_parity( + case: QualificationCase, + context: RuntimeContext, + definition: Mapping[str, Any], + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or not requests: + raise QualificationError("text-generation dataset must contain a non-empty requests list") + selected_requests, selection = _select_dataset_rows( + requests, definition, sample_limit, "text-generation" + ) + selected = [] + for index, request in enumerate(selected_requests): + if not isinstance(request, Mapping): + raise QualificationError(f"text-generation request {index} must be an object") + selected.append( + { + "sample_id": str( + request.get("id") or request.get("sample_id") or f"sample-{index}" + ), + "prompt": _prompt(request), + } + ) + reference = configured.get("reference", {}) + candidate_request = configured.get("request", {}) + if not isinstance(reference, Mapping) or not isinstance(candidate_request, Mapping): + raise QualificationError("Accuracy reference and request must be objects") + reference_task = str(reference.get("task", "causal-lm")) + if reference_task not in {"causal-lm", "seq2seq-lm"}: + raise QualificationError(f"unsupported Accuracy reference task {reference_task!r}") + output_token_policy = str(reference.get("output_token_policy", "new-tokens")) + if output_token_policy not in {"new-tokens", "strip-start", "strip-start-and-eos"}: + raise QualificationError( + f"unsupported Accuracy output token policy {output_token_policy!r}" + ) + reference_generation = dict(candidate_request) + source_language_placement = reference.get("source_language_placement") + if source_language_placement is not None: + reference_generation["source_language_placement"] = source_language_placement + reference_model = reference.get("model", case.candidate["checkpoint"]) + if not isinstance(reference_model, str) or not reference_model: + raise QualificationError("accuracy.reference.model must be a non-empty string") + reference_revision = reference.get("revision") + if reference_revision is None and reference_model == case.candidate["checkpoint"]: + reference_revision = case.candidate.get("revision") + if reference_revision is not None and ( + not isinstance(reference_revision, str) or not reference_revision + ): + raise QualificationError("accuracy.reference.revision must be a non-empty string") + trust_remote_code = reference.get( + "trust_remote_code", case.candidate.get("trust_remote_code", False) + ) + if not isinstance(trust_remote_code, bool): + raise QualificationError("accuracy.reference.trust_remote_code must be a boolean") + reference_request = { + "model": reference_model, + "revision": reference_revision, + "trust_remote_code": trust_remote_code, + "precision": str(reference.get("precision", "fp32")), + "task": reference_task, + "output_token_policy": output_token_policy, + "prompt_token_limit": int(configured.get("prompt_token_limit", 192)), + "truncation_side": str(configured.get("truncation_side", "left")), + "generation": reference_generation, + "samples": selected, + } + experts_implementation = reference.get("experts_implementation") + if experts_implementation is not None: + if not isinstance(experts_implementation, str) or not experts_implementation: + raise QualificationError( + "accuracy.reference.experts_implementation must be a non-empty string" + ) + reference_request["experts_implementation"] = experts_implementation + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + runner_value = reference.get("command") + if runner_value is None: + runner = ( + context.repository / "qualification_tests/benchmark_qualification/references/hf_text_generation.py" + ) + else: + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("text-generation reference.command must be a string") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError( + f"text-generation reference runner is not family-owned: {runner}" + ) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"HF Accuracy reference failed; see {output}") + reference_samples = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(reference_samples, list) or len(reference_samples) != len(selected): + raise QualificationError("HF Accuracy reference returned an invalid sample set") + + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(candidate_request), "prompt": sample["prompt"]}, + } + for sample in reference_samples + ] + candidate, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + rows = [] + for actual, expected in zip(candidate, reference_samples, strict=True): + actual_ids = actual.get("token_ids") + expected_ids = expected.get("token_ids") + matched = actual_ids == expected_ids and isinstance(actual_ids, list) and bool(actual_ids) + rows.append( + { + "sample_id": expected["sample_id"], + "passed": matched, + "candidate_token_ids": actual_ids, + "reference_token_ids": expected_ids, + } + ) + gates = configured.get("gate", {}) + if not isinstance(gates, Mapping): + raise QualificationError("Accuracy gate must be an object") + minimum_rate = float(gates.get("min_pass_rate", 1.0)) + allowed_failures = int(gates.get("allowed_failures", 0)) + passed_count = sum(bool(row["passed"]) for row in rows) + failed_count = len(rows) - passed_count + pass_rate = passed_count / len(rows) + passed = pass_rate >= minimum_rate and failed_count <= allowed_failures + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": {**dataset.receipt(), "selection": selection}, + "metrics": { + "samples": len(rows), + "passed_samples": passed_count, + "failed_samples": failed_count, + "continuation_token_pass_rate": pass_rate, + }, + "gate": {"min_pass_rate": minimum_rate, "allowed_failures": allowed_failures}, + "samples": rows, + } + + +def _encoder_embedding_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + prompt_prefix = configured.get("prompt_prefix", "") + if not isinstance(prompt_prefix, str): + raise QualificationError("accuracy.prompt_prefix must be a string") + samples = _sts_samples(dataset.path, sample_limit, prompt_prefix) + reference = configured.get("reference", {}) + gates = configured.get("gate", {}) + if not isinstance(reference, Mapping) or not isinstance(gates, Mapping): + raise QualificationError("encoder Accuracy reference and gate must be objects") + task = str(case.candidate["task"]) + try: + mode, operation = { + "encoding": ("cls", "encode"), + "embedding": ("embedding", "embed"), + }[task] + except KeyError as error: + raise QualificationError( + f"embedding vector parity does not support candidate task {task!r}" + ) from error + model_class = reference.get("model_class", "auto") + tokenizer_class = reference.get("tokenizer_class", "auto") + if not all(isinstance(value, str) and value for value in (model_class, tokenizer_class)): + raise QualificationError("encoder reference classes must be non-empty strings") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "trust_remote_code": bool(case.candidate.get("trust_remote_code", False)), + "precision": str(reference.get("precision", "fp32")), + "mode": mode, + "model_class": model_class, + "tokenizer_class": tokenizer_class, + "max_length": int(case.candidate.get("build", {}).get("max_sequence_length", 512)), + "samples": samples, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + output.mkdir(parents=True, exist_ok=True) + _json(request_path, reference_request) + runner = context.repository / "qualification_tests/benchmark_qualification/references/hf_encoder.py" + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"HF encoder Accuracy reference failed; see {output}") + reference_samples = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(reference_samples, list) or len(reference_samples) != len(samples): + raise QualificationError("HF encoder Accuracy reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"prompt": sample["prompt"]}, + } + for sample in samples + ] + candidate, bundle = _candidate_outputs(case, context, output, operation, candidate_requests) + compared = _compare_encoder_embeddings(reference_samples, candidate, gates) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": compared["status"], + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": compared["metrics"], + "gate": compared["gate"], + "samples": compared["samples"], + "pairs": compared["pairs"], + } + + +def _reranking_score_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + samples = _sts_reranking_samples(dataset.path, sample_limit) + reference = configured.get("reference", {}) + gate = configured.get("gate", {}) + if not isinstance(reference, Mapping) or not isinstance(gate, Mapping): + raise QualificationError("reranking reference and gate must be objects") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("reranking reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"reranking reference runner is not family-owned: {runner}") + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json( + request_path, + { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "samples": samples, + }, + ) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"reranking Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(samples): + raise QualificationError("reranking reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"query": sample["query"], "documents": sample["documents"]}, + } + for sample in samples + ] + actual, bundle = _candidate_outputs(case, context, output, "rerank", candidate_requests) + maximum_error = float(gate.get("max_score_abs_error", 0.05)) + minimum_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if maximum_error < 0.0 or not math.isfinite(maximum_error): + raise QualificationError("max_score_abs_error must be finite and nonnegative") + if not 0.0 <= minimum_rate <= 1.0: + raise QualificationError("min_sample_pass_rate must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(samples, actual, expected, strict=True): + candidate_scores = _reranking_scores(candidate_sample, "candidate") + reference_scores = _reranking_scores(reference_sample, "reference") + if len(candidate_scores) != len(reference_scores) or len(reference_scores) < 2: + raise QualificationError("reranking reference and candidate score counts differ") + errors = [ + abs(left - right) + for left, right in zip(candidate_scores, reference_scores, strict=True) + ] + candidate_order = _reranking_order(candidate_scores) + reference_order = _reranking_order(reference_scores) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": max(errors) <= maximum_error and candidate_order == reference_order, + "max_score_abs_error": max(errors), + "candidate_order": candidate_order, + "reference_order": reference_order, + } + ) + pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = pass_rate >= minimum_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": pass_rate, + "max_score_abs_error": max(row["max_score_abs_error"] for row in rows), + }, + "gate": { + "max_score_abs_error": maximum_error, + "min_sample_pass_rate": minimum_rate, + }, + "samples": rows, + } + + +def _reranking_scores(summary: Mapping[str, Any], label: str) -> list[float]: + values = summary.get("scores") + if not isinstance(values, list) or not values: + raise QualificationError(f"{label} reranking output has no scores") + scores = [float(value) for value in values] + if not all(math.isfinite(value) for value in scores): + raise QualificationError(f"{label} reranking output has non-finite scores") + return scores + + +def _reranking_order(scores: Sequence[float]) -> list[int]: + return sorted(range(len(scores)), key=lambda index: (-scores[index], index)) + + +def _sts_reranking_samples(path: Path, count: int) -> list[dict[str, Any]]: + pairs = [] + with path.open(encoding="utf-8") as stream: + for dataset_index, line in enumerate(stream): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise QualificationError( + f"STSBenchmark row {dataset_index} is not valid JSON" + ) from error + if not isinstance(row, Mapping): + raise QualificationError(f"STSBenchmark row {dataset_index} must be an object") + sentence1 = row.get("sentence1") + sentence2 = row.get("sentence2") + if not all( + isinstance(value, str) and value.strip() for value in (sentence1, sentence2) + ): + raise QualificationError( + f"STSBenchmark row {dataset_index} must contain two sentences" + ) + pairs.append((str(sentence1).strip(), str(sentence2).strip())) + if len(pairs) == count * 2: + break + if len(pairs) != count * 2: + raise QualificationError( + f"STSBenchmark contains {len(pairs)} usable rows; {count * 2} required" + ) + return [ + { + "sample_id": f"stsbenchmark-rerank-{index:06d}", + "query": pairs[index * 2][0], + "documents": [pairs[index * 2][1], pairs[index * 2 + 1][1]], + } + for index in range(count) + ] + + +def _sts_samples(path: Path, count: int, prefix: str) -> list[dict[str, Any]]: + pairs: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as stream: + for dataset_index, line in enumerate(stream): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise QualificationError( + f"STSBenchmark row {dataset_index} is not valid JSON" + ) from error + if not isinstance(row, Mapping): + raise QualificationError(f"STSBenchmark row {dataset_index} must be an object") + sentence1 = row.get("sentence1") + sentence2 = row.get("sentence2") + if not all( + isinstance(value, str) and value.strip() for value in (sentence1, sentence2) + ): + raise QualificationError( + f"STSBenchmark row {dataset_index} must contain two sentences" + ) + try: + score = float(row["score"]) + except (KeyError, TypeError, ValueError) as error: + raise QualificationError( + f"STSBenchmark row {dataset_index} has an invalid score" + ) from error + if not math.isfinite(score): + raise QualificationError(f"STSBenchmark row {dataset_index} has a non-finite score") + pair_id = f"stsbenchmark-{dataset_index:06d}" + for suffix, side, sentence in ( + ("a", "sentence1", sentence1), + ("b", "sentence2", sentence2), + ): + pairs.append( + { + "sample_id": f"{pair_id}-{suffix}", + "pair_id": pair_id, + "pair_side": side, + "score": score, + "prompt": prefix + str(sentence).strip(), + } + ) + if len(pairs) == count * 2: + break + if len(pairs) != count * 2: + raise QualificationError( + f"STSBenchmark contains {len(pairs) // 2} usable pairs; {count} required" + ) + return pairs + + +def _compare_encoder_embeddings( + reference: Sequence[Mapping[str, Any]], + candidate: Sequence[Mapping[str, Any]], + gates: Mapping[str, Any], +) -> dict[str, Any]: + if len(reference) != len(candidate) or not reference: + raise QualificationError("encoder reference and candidate sample counts differ") + minimum_cosine = float(gates.get("min_vector_cosine", 0.999)) + minimum_rate = float(gates.get("min_vector_pass_rate", 1.0)) + maximum_pair_delta = float(gates.get("max_pair_cosine_abs_delta", 0.02)) + if not -1.0 <= minimum_cosine <= 1.0: + raise QualificationError("min_vector_cosine must be in [-1, 1]") + if not 0.0 <= minimum_rate <= 1.0: + raise QualificationError("min_vector_pass_rate must be in [0, 1]") + if maximum_pair_delta < 0.0 or not math.isfinite(maximum_pair_delta): + raise QualificationError("max_pair_cosine_abs_delta must be finite and nonnegative") + + sample_rows: list[dict[str, Any]] = [] + vector_cosines: list[float] = [] + reference_pairs: dict[str, dict[str, tuple[Mapping[str, Any], list[float]]]] = {} + candidate_pairs: dict[str, dict[str, list[float]]] = {} + for index, (expected, actual) in enumerate(zip(reference, candidate, strict=True)): + expected_id = str(expected.get("sample_id", index)) + vector = expected.get("vector") + if not isinstance(vector, list): + raise QualificationError(f"HF encoder sample {expected_id!r} has no vector") + reference_vector = _finite_vector(vector, f"HF encoder sample {expected_id!r}") + candidate_vector = _candidate_vector(actual, expected_id) + cosine = _vector_cosine(reference_vector, candidate_vector) + pair_id = str(expected.get("pair_id", "")) + pair_side = str(expected.get("pair_side", "")) + if not pair_id or pair_side not in {"sentence1", "sentence2"}: + raise QualificationError(f"HF encoder sample {expected_id!r} has invalid pair metadata") + reference_pairs.setdefault(pair_id, {})[pair_side] = (expected, reference_vector) + candidate_pairs.setdefault(pair_id, {})[pair_side] = candidate_vector + vector_cosines.append(cosine) + sample_rows.append( + { + "sample_id": expected_id, + "pair_id": pair_id, + "pair_side": pair_side, + "vector_dim": len(reference_vector), + "vector_cosine": cosine, + "passed": cosine >= minimum_cosine, + } + ) + + pair_rows: list[dict[str, Any]] = [] + pair_deltas: list[float] = [] + scores: list[float] = [] + reference_similarities: list[float] = [] + candidate_similarities: list[float] = [] + for pair_id, expected in reference_pairs.items(): + actual = candidate_pairs.get(pair_id, {}) + if set(expected) != {"sentence1", "sentence2"} or set(actual) != { + "sentence1", + "sentence2", + }: + raise QualificationError(f"encoder pair {pair_id!r} is incomplete") + reference_similarity = _vector_cosine(expected["sentence1"][1], expected["sentence2"][1]) + candidate_similarity = _vector_cosine(actual["sentence1"], actual["sentence2"]) + delta = abs(candidate_similarity - reference_similarity) + score = float(expected["sentence1"][0]["score"]) + pair_deltas.append(delta) + scores.append(score) + reference_similarities.append(reference_similarity) + candidate_similarities.append(candidate_similarity) + pair_rows.append( + { + "pair_id": pair_id, + "score": score, + "hf_cosine": reference_similarity, + "candidate_cosine": candidate_similarity, + "cosine_abs_delta": delta, + "passed": delta <= maximum_pair_delta, + } + ) + + passed_vectors = sum(value >= minimum_cosine for value in vector_cosines) + pass_rate = passed_vectors / len(vector_cosines) + max_pair_delta = max(pair_deltas) + status = ( + "passed" if pass_rate >= minimum_rate and max_pair_delta <= maximum_pair_delta else "failed" + ) + return { + "status": status, + "metrics": { + "samples": len(vector_cosines), + "pairs": len(pair_rows), + "passed_vectors": passed_vectors, + "vector_pass_rate": pass_rate, + "mean_vector_cosine": math.fsum(vector_cosines) / len(vector_cosines), + "min_vector_cosine": min(vector_cosines), + "mean_pair_cosine_abs_delta": math.fsum(pair_deltas) / len(pair_deltas), + "max_pair_cosine_abs_delta": max_pair_delta, + "hf_sts_spearman": _spearman(scores, reference_similarities), + "candidate_sts_spearman": _spearman(scores, candidate_similarities), + }, + "gate": { + "min_vector_cosine": minimum_cosine, + "min_vector_pass_rate": minimum_rate, + "max_pair_cosine_abs_delta": maximum_pair_delta, + }, + "samples": sample_rows, + "pairs": pair_rows, + } + + +def _candidate_vector(summary: Mapping[str, Any], sample_id: str) -> list[float]: + values = summary.get("values") + if not isinstance(values, list): + raise QualificationError(f"TRTMC encoder sample {sample_id!r} has no vector values") + vector = _finite_vector(values, f"TRTMC encoder sample {sample_id!r}") + dim = summary.get("dim") + if isinstance(dim, bool) or not isinstance(dim, int) or dim < 1: + dim = len(vector) + if summary.get("feature_kind") == "token": + if len(vector) < dim: + raise QualificationError(f"TRTMC encoder sample {sample_id!r} is shorter than dim") + vector = vector[:dim] + elif len(vector) != dim: + raise QualificationError( + f"TRTMC encoder sample {sample_id!r} has {len(vector)} values for dim {dim}" + ) + return vector + + +def _finite_vector(values: Sequence[Any], label: str) -> list[float]: + try: + vector = [float(value) for value in values] + except (TypeError, ValueError) as error: + raise QualificationError(f"{label} contains a non-numeric value") from error + if not vector or any(not math.isfinite(value) for value in vector): + raise QualificationError(f"{label} must be non-empty and finite") + return vector + + +def _vector_cosine(left: Sequence[float], right: Sequence[float]) -> float: + if len(left) != len(right) or not left: + raise QualificationError(f"encoder vector dimensions differ: {len(left)} != {len(right)}") + left_norm = math.sqrt(math.fsum(value * value for value in left)) + right_norm = math.sqrt(math.fsum(value * value for value in right)) + if left_norm <= 0.0 or right_norm <= 0.0: + raise QualificationError("encoder vector must have a nonzero norm") + cosine = math.fsum(a * b for a, b in zip(left, right, strict=True)) / (left_norm * right_norm) + return max(-1.0, min(1.0, cosine)) + + +def _spearman(left: Sequence[float], right: Sequence[float]) -> float | None: + if len(left) != len(right) or len(left) < 2: + return None + return _pearson(_ranks(left), _ranks(right)) + + +def _ranks(values: Sequence[float]) -> list[float]: + indexed = sorted(enumerate(values), key=lambda item: item[1]) + ranks = [0.0] * len(values) + start = 0 + while start < len(indexed): + end = start + 1 + while end < len(indexed) and indexed[end][1] == indexed[start][1]: + end += 1 + rank = (start + end - 1) / 2.0 + for position in range(start, end): + ranks[indexed[position][0]] = rank + start = end + return ranks + + +def _pearson(left: Sequence[float], right: Sequence[float]) -> float | None: + left_mean = math.fsum(left) / len(left) + right_mean = math.fsum(right) / len(right) + centered_left = [value - left_mean for value in left] + centered_right = [value - right_mean for value in right] + denominator = math.sqrt( + math.fsum(value * value for value in centered_left) + * math.fsum(value * value for value in centered_right) + ) + if denominator <= 0.0: + return None + return ( + math.fsum(a * b for a, b in zip(centered_left, centered_right, strict=True)) / denominator + ) + + +def _etth1( + case: QualificationCase, + context: RuntimeContext, + definition: Mapping[str, Any], + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + window = configured.get("window") + reference = configured.get("reference") + if not isinstance(window, Mapping) or not isinstance(reference, Mapping): + raise QualificationError("ETTh1 window and reference must be objects") + samples = _etth1_windows(dataset.path, definition, window, int(configured.get("samples", 10))) + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("ETTh1 reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + if not runner.is_file(): + raise QualificationError(f"ETTh1 reference runner does not exist: {runner}") + request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "samples": samples, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"ETTh1 Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(samples): + raise QualificationError("ETTh1 reference returned an invalid sample set") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + "past_values": [float(value) for value in sample["past_values"]], + "observed_mask": [1.0] * len(sample["past_values"]), + "frequency": int(sample.get("frequency", 0)), + }, + } + for sample in samples + ] + actual, bundle = _candidate_outputs(case, context, output, "solve", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("ETTh1 gate must be an object") + relative_limit = float(gate["max_relative_l2"]) + absolute_limit = float(gate["max_absolute_error"]) + rows = [] + for left, right in zip(actual, expected, strict=True): + left_values = [float(value) for value in left["values"]] + right_values = [float(value) for value in right["values"]] + shape_match = left.get("shape") == right.get("shape") and len(left_values) == len( + right_values + ) + if shape_match: + difference = math.sqrt( + sum((a - b) ** 2 for a, b in zip(left_values, right_values, strict=True)) + ) + norm = math.sqrt(sum(value**2 for value in right_values)) + relative_l2 = difference / max(norm, 1.0e-12) + maximum = max(abs(a - b) for a, b in zip(left_values, right_values, strict=True)) + else: + relative_l2 = math.inf + maximum = math.inf + rows.append( + { + "sample_id": right["sample_id"], + "passed": shape_match + and relative_l2 <= relative_limit + and maximum <= absolute_limit, + "shape_match": shape_match, + "relative_l2": relative_l2, + "max_absolute_error": maximum, + } + ) + passed = all(row["passed"] for row in rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "failed_samples": sum(not row["passed"] for row in rows), + "max_relative_l2": max(row["relative_l2"] for row in rows), + "max_absolute_error": max(row["max_absolute_error"] for row in rows), + }, + "gate": { + "max_relative_l2": relative_limit, + "max_absolute_error": absolute_limit, + }, + "samples": rows, + } + + +def _candidate_outputs( + case: QualificationCase, + context: RuntimeContext, + output: Path, + operation: str, + requests: Sequence[Mapping[str, Any]], +) -> tuple[list[dict[str, Any]], Path]: + if not requests: + raise QualificationError("Accuracy candidate has no requests") + worker, runtime_root = require_candidate(context) + first_request = requests[0].get("request") + if not isinstance(first_request, Mapping): + raise QualificationError("Accuracy candidate request must be an object") + descriptor = write_model_descriptor(case, output, first_request, context=context) + bundle = prepare_bundle(case, context, output, descriptor) + data_path = output / "candidate-inputs.json" + _json(data_path, list(requests)) + candidate_output = output / "candidate-run" + command = [ + str(context.trtmc_bench), + "run", + "--model", + str(descriptor), + "--case", + case.name, + "--operation", + operation, + "--bundle", + str(bundle), + "--data", + str(data_path), + "--runtime-root", + str(runtime_root), + "--worker", + str(worker), + "--warmup", + "0", + "--iterations", + "1", + "--telemetry", + "off", + "--no-build", + "--output", + str(candidate_output), + ] + completed = run_command( + command, + output, + "candidate", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"TRTMC Accuracy candidate failed; see {output}") + result = json.loads((candidate_output / "result.json").read_text(encoding="utf-8")) + cells = result.get("cells") + if not isinstance(cells, list) or len(cells) != len(requests): + raise QualificationError("trtmc-bench returned an invalid Accuracy result set") + outputs = [] + candidate_root = candidate_output.resolve() + + def resolve_artifact(artifact_root: Path, value: str) -> str: + path = Path(value) + resolved = path.resolve() if path.is_absolute() else (artifact_root / path).resolve() + if not resolved.is_relative_to(artifact_root): + raise QualificationError("trtmc-bench returned an unsafe artifact path") + return str(resolved) + + for cell in cells: + if not isinstance(cell, Mapping) or cell.get("status") != "completed": + raise QualificationError("trtmc-bench reported a failed Accuracy request") + summary = cell.get("output_summary") + if not isinstance(summary, Mapping): + raise QualificationError("trtmc-bench omitted an Accuracy output") + resolved_summary = dict(summary) + artifact_dir = cell.get("artifact_dir") + if isinstance(artifact_dir, str) and artifact_dir: + artifact_root = (candidate_output / artifact_dir).resolve() + if not artifact_root.is_relative_to(candidate_root): + raise QualificationError("trtmc-bench returned an unsafe artifact directory") + for name, value in tuple(resolved_summary.items()): + if name.endswith("_artifact") and isinstance(value, str): + resolved_summary[name] = resolve_artifact(artifact_root, value) + elif name.endswith("_artifacts") and isinstance(value, list): + resolved_summary[name] = [ + resolve_artifact(artifact_root, item) + for item in value + if isinstance(item, str) + ] + outputs.append(resolved_summary) + return outputs, bundle + + +def _image_parity_samples(dataset: Dataset, sample_limit: int, task: str) -> list[dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError(f"{task} dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"{task} dataset requires at least {sample_limit} requests") + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + relative = request.get("image") if isinstance(request, Mapping) else None + if not isinstance(relative, str) or not relative: + raise QualificationError(f"{task} request {index} has no image") + image = (root / relative).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError(f"{task} request {index} image is unavailable: {image}") + sample = { + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + } + label_name = request.get("label_name") + if isinstance(label_name, str) and label_name.strip(): + sample["label_name"] = label_name.strip() + selected.append(sample) + return selected + + +def _family_image_reference( + case: QualificationCase, + context: RuntimeContext, + output: Path, + selected: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + reference = case.values.get("reference") + if not isinstance(reference, Mapping): + raise QualificationError("image Accuracy reference must be an object") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("image Accuracy reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"image reference runner is not family-owned: {runner}") + request = case.values.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("image Accuracy request must be an object") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "request": dict(request), + "samples": list(selected), + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"image Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("image Accuracy reference returned an invalid sample set") + return expected + + +def _box_iou(left: Sequence[float], right: Sequence[float]) -> float: + intersection_width = max(0.0, min(left[2], right[2]) - max(left[0], right[0])) + intersection_height = max(0.0, min(left[3], right[3]) - max(left[1], right[1])) + intersection = intersection_width * intersection_height + left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1]) + right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1]) + union = left_area + right_area - intersection + return intersection / union if union > 0.0 else 0.0 + + +def _detections(value: Mapping[str, Any], label: str) -> list[dict[str, Any]]: + boxes = value.get("boxes") + scores = value.get("scores") + classes = value.get("class_ids", value.get("classes")) + if not isinstance(boxes, list) or not isinstance(scores, list) or not isinstance(classes, list): + raise QualificationError(f"{label} detection output is incomplete") + if boxes and isinstance(boxes[0], list): + rows = boxes + else: + if len(boxes) % 4: + raise QualificationError(f"{label} detection boxes are not xyxy rows") + rows = [boxes[index : index + 4] for index in range(0, len(boxes), 4)] + if len(rows) != len(scores) or len(rows) != len(classes): + raise QualificationError(f"{label} detection arrays have different lengths") + detections = [] + for index, (box, score, class_id) in enumerate(zip(rows, scores, classes, strict=True)): + if ( + not isinstance(box, list) + or len(box) != 4 + or isinstance(class_id, bool) + or not isinstance(class_id, int) + ): + raise QualificationError(f"{label} detection {index} is invalid") + numeric_box = [float(coordinate) for coordinate in box] + numeric_score = float(score) + if not all(math.isfinite(value) for value in (*numeric_box, numeric_score)): + raise QualificationError(f"{label} detection {index} is not finite") + detections.append({"box": numeric_box, "score": numeric_score, "class_id": class_id}) + return detections + + +def _coco_detection_samples( + dataset: Dataset, sample_limit: int +) -> tuple[list[dict[str, Any]], str]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("COCO object-detection dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError( + f"COCO object-detection dataset requires at least {sample_limit} requests" + ) + root = dataset.path.parent.resolve() + selected = [] + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"COCO object-detection request {index} is invalid") + relative = request.get("image") + annotations = request.get("annotations") + if not isinstance(relative, str) or not relative: + raise QualificationError(f"COCO object-detection request {index} has no image") + if not isinstance(annotations, list) or not annotations: + raise QualificationError( + f"COCO object-detection request {index} has no ground-truth annotations" + ) + image = (root / relative).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError( + f"COCO object-detection request {index} image is unavailable: {image}" + ) + ground_truth = [] + for annotation_index, annotation in enumerate(annotations): + if not isinstance(annotation, Mapping): + raise QualificationError( + f"COCO object-detection annotation {index}:{annotation_index} is invalid" + ) + box = annotation.get("bbox_xyxy") + category_id = annotation.get("category_id") + category_index = annotation.get("category_index") + if ( + not isinstance(box, list) + or len(box) != 4 + or any(not isinstance(value, (int, float)) for value in box) + or isinstance(category_id, bool) + or not isinstance(category_id, int) + or isinstance(category_index, bool) + or not isinstance(category_index, int) + ): + raise QualificationError( + f"COCO object-detection annotation {index}:{annotation_index} is invalid" + ) + numeric_box = [float(value) for value in box] + if ( + not all(math.isfinite(value) for value in numeric_box) + or numeric_box[2] <= numeric_box[0] + or numeric_box[3] <= numeric_box[1] + ): + raise QualificationError( + f"COCO object-detection annotation {index}:{annotation_index} has an invalid box" + ) + ground_truth.append( + { + "box": numeric_box, + "category_id": category_id, + "category_index": category_index, + } + ) + selected.append( + { + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + "annotations": ground_truth, + } + ) + sampling = payload.get("sampling") + return selected, str(sampling) if isinstance(sampling, str) else "fixed manifest order" + + +def _coco_ap_at_iou( + selected: Sequence[Mapping[str, Any]], + outputs: Sequence[Mapping[str, Any]], + label_field: str, + iou_threshold: float, +) -> float: + ground_truth: dict[int, dict[str, list[Sequence[float]]]] = {} + predictions: dict[int, list[tuple[float, str, Sequence[float]]]] = {} + for sample, output in zip(selected, outputs, strict=True): + sample_id = str(sample["sample_id"]) + for annotation in sample["annotations"]: + class_id = int(annotation[label_field]) + ground_truth.setdefault(class_id, {}).setdefault(sample_id, []).append( + annotation["box"] + ) + for detection in sorted( + _detections(output, "object-detection output"), + key=lambda value: -float(value["score"]), + )[:100]: + predictions.setdefault(int(detection["class_id"]), []).append( + (float(detection["score"]), sample_id, detection["box"]) + ) + + category_aps = [] + for class_id, sample_boxes in ground_truth.items(): + total_ground_truth = sum(len(boxes) for boxes in sample_boxes.values()) + matched: dict[str, set[int]] = defaultdict(set) + true_positives = false_positives = 0 + precisions = [] + recalls = [] + for _, sample_id, box in sorted(predictions.get(class_id, []), key=lambda value: -value[0]): + candidates = [ + (_box_iou(box, expected), index) + for index, expected in enumerate(sample_boxes.get(sample_id, [])) + if index not in matched[sample_id] + ] + if candidates and max(candidates)[0] >= iou_threshold: + _, matched_index = max(candidates) + matched[sample_id].add(matched_index) + true_positives += 1 + else: + false_positives += 1 + precisions.append(true_positives / (true_positives + false_positives)) + recalls.append(true_positives / total_ground_truth) + category_aps.append( + sum( + max( + ( + precision + for precision, recall in zip(precisions, recalls, strict=True) + if recall >= point + ), + default=0.0, + ) + for point in (index / 100.0 for index in range(101)) + ) + / 101.0 + ) + if not category_aps: + raise QualificationError("COCO object-detection subset has no ground truth") + return sum(category_aps) / len(category_aps) + + +def _coco_detection_metrics( + selected: Sequence[Mapping[str, Any]], + outputs: Sequence[Mapping[str, Any]], + label_field: str, +) -> dict[str, float]: + if len(selected) != len(outputs): + raise QualificationError("COCO object-detection output count differs from the dataset") + thresholds = [0.5 + index * 0.05 for index in range(10)] + average_precisions = [ + _coco_ap_at_iou(selected, outputs, label_field, threshold) for threshold in thresholds + ] + return { + "map_50_95": sum(average_precisions) / len(average_precisions), + "map_50": average_precisions[0], + } + + +def _coco_object_detection_accuracy( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected, sampling = _coco_detection_samples(dataset, sample_limit) + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("COCO object-detection request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "detect", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("COCO object-detection gate must be an object") + label_spaces = { + "coco-category-id": "category_id", + "coco-contiguous-80": "category_index", + } + label_space = configured.get("label_space") + if label_space not in label_spaces: + raise QualificationError( + "COCO object-detection label_space must be coco-category-id or coco-contiguous-80" + ) + map_50_95_limit = float(gate.get("max_map_50_95_drop", 0.02)) + map_50_limit = float(gate.get("max_map_50_drop", 0.02)) + if not all(math.isfinite(value) and value >= 0.0 for value in (map_50_95_limit, map_50_limit)): + raise QualificationError("COCO object-detection mAP drops must be finite and nonnegative") + label_field = label_spaces[str(label_space)] + candidate_metrics = _coco_detection_metrics(selected, actual, label_field) + reference_metrics = _coco_detection_metrics(selected, expected, label_field) + map_50_95_drop = reference_metrics["map_50_95"] - candidate_metrics["map_50_95"] + map_50_drop = reference_metrics["map_50"] - candidate_metrics["map_50"] + passed = map_50_95_drop <= map_50_95_limit and map_50_drop <= map_50_limit + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": {**dataset.receipt(), "sampling": sampling}, + "metrics": { + "samples": len(selected), + "label_space": label_space, + "candidate_map_50_95": candidate_metrics["map_50_95"], + "reference_map_50_95": reference_metrics["map_50_95"], + "map_50_95_drop": map_50_95_drop, + "candidate_map_50": candidate_metrics["map_50"], + "reference_map_50": reference_metrics["map_50"], + "map_50_drop": map_50_drop, + }, + "gate": { + "max_map_50_95_drop": map_50_95_limit, + "max_map_50_drop": map_50_limit, + }, + } + + +def _semantic_mask(value: Mapping[str, Any], label: str) -> tuple[int, int, list[int]]: + height = value.get("height") + width = value.get("width") + mask = value.get("mask") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or not isinstance(mask, list) + or len(mask) != height * width + ): + raise QualificationError(f"{label} semantic-segmentation output is invalid") + if any(isinstance(item, bool) or not isinstance(item, int) for item in mask): + raise QualificationError(f"{label} semantic-segmentation mask is not integral") + return height, width, mask + + +def _binary_masks(value: Mapping[str, Any], label: str) -> tuple[int, int, list[list[bool]]]: + height = value.get("height") + width = value.get("width") + count = value.get("num_masks") + masks = value.get("masks") + kind = value.get("mask_kind", "logits") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or isinstance(count, bool) + or not isinstance(count, int) + or count < 1 + or not isinstance(masks, list) + or len(masks) != count * height * width + or kind not in {"logits", "probability", "binary"} + ): + raise QualificationError(f"{label} prompted-segmentation output is invalid") + try: + numeric = [float(value) for value in masks] + except (TypeError, ValueError) as error: + raise QualificationError(f"{label} prompted-segmentation masks are not numeric") from error + if not all(math.isfinite(value) for value in numeric): + raise QualificationError(f"{label} prompted-segmentation masks are not finite") + threshold = 0.0 if kind == "logits" else 0.5 + area = height * width + return ( + height, + width, + [ + [value > threshold for value in numeric[index : index + area]] + for index in range(0, len(numeric), area) + ], + ) + + +def _mask_iou(left: Sequence[bool], right: Sequence[bool]) -> float: + intersection = sum(a and b for a, b in zip(left, right, strict=True)) + union = sum(a or b for a, b in zip(left, right, strict=True)) + return intersection / union if union else 1.0 + + +def _match_masks( + candidate: Sequence[Sequence[bool]], reference: Sequence[Sequence[bool]] +) -> list[float]: + matched_reference: set[int] = set() + matches = [] + for mask in candidate: + choices = [ + (_mask_iou(mask, expected), index) + for index, expected in enumerate(reference) + if index not in matched_reference + ] + if not choices: + continue + iou, index = max(choices) + matched_reference.add(index) + matches.append(iou) + return matches + + +def _prompted_segmentation_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "prompted-segmentation") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("prompted-segmentation request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs( + case, context, output, "segment_prompted", candidate_requests + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("prompted-segmentation gate must be an object") + minimum_iou = float(gate.get("min_mask_iou", 0.7)) + minimum_match_rate = float(gate.get("min_mask_match_rate", 1.0)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 for value in (minimum_iou, minimum_match_rate, minimum_sample_rate) + ): + raise QualificationError("prompted-segmentation gates must be in [0, 1]") + + rows = [] + total_matches = total_masks = 0 + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + left_height, left_width, left = _binary_masks(candidate_sample, "candidate") + right_height, right_width, right = _binary_masks(reference_sample, "reference") + matches = ( + _match_masks(left, right) + if (left_height, left_width) == (right_height, right_width) + else [] + ) + denominator = max(len(left), len(right), 1) + match_rate = len(matches) / denominator + minimum_sample_iou = min(matches, default=0.0) + passed = match_rate >= minimum_match_rate and minimum_sample_iou >= minimum_iou + rows.append( + { + "sample_id": sample["sample_id"], + "passed": passed, + "candidate_masks": len(left), + "reference_masks": len(right), + "mask_match_rate": match_rate, + "min_mask_iou": minimum_sample_iou, + } + ) + total_matches += len(matches) + total_masks += denominator + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "mask_match_rate": total_matches / total_masks, + "min_mask_iou": min(row["min_mask_iou"] for row in rows), + }, + "gate": { + "min_mask_iou": minimum_iou, + "min_mask_match_rate": minimum_match_rate, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _instance_masks( + value: Mapping[str, Any], label: str +) -> tuple[int, int, list[list[bool]], list[float], list[list[float]]]: + height, width, masks = _binary_masks(value, label) + scores = value.get("iou_scores") + boxes = value.get("boxes") + if ( + not isinstance(scores, list) + or len(scores) != len(masks) + or not isinstance(boxes, list) + or len(boxes) != len(masks) + or value.get("box_coordinates") != "original_image_pixels_xyxy" + ): + raise QualificationError(f"{label} instance output is incomplete") + numeric_scores = [float(score) for score in scores] + numeric_boxes = [] + for box in boxes: + if not isinstance(box, list) or len(box) != 4: + raise QualificationError(f"{label} instance box is invalid") + numeric_boxes.append([float(coordinate) for coordinate in box]) + if not all( + math.isfinite(number) + for number in (*numeric_scores, *(value for box in numeric_boxes for value in box)) + ): + raise QualificationError(f"{label} instance scores or boxes are not finite") + return height, width, masks, numeric_scores, numeric_boxes + + +def _match_instances( + candidate: tuple[int, int, list[list[bool]], list[float], list[list[float]]], + reference: tuple[int, int, list[list[bool]], list[float], list[list[float]]], +) -> list[dict[str, float]]: + if candidate[:2] != reference[:2]: + return [] + matched_reference: set[int] = set() + matches = [] + for index, mask in enumerate(candidate[2]): + choices = [ + (_mask_iou(mask, expected), expected_index) + for expected_index, expected in enumerate(reference[2]) + if expected_index not in matched_reference + ] + if not choices: + continue + mask_iou, expected_index = max(choices) + matched_reference.add(expected_index) + matches.append( + { + "mask_iou": mask_iou, + "box_iou": _box_iou(candidate[4][index], reference[4][expected_index]), + "score_abs_error": abs(candidate[3][index] - reference[3][expected_index]), + } + ) + return matches + + +def _text_prompted_instance_segmentation_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "text-prompted-segmentation") + if any(not sample.get("label_name") for sample in selected): + raise QualificationError("text-prompted segmentation samples require label_name") + expected = _family_image_reference(case, context, output, selected) + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + "image_path": sample["image_path"], + "prompt": sample["label_name"], + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs( + case, context, output, "segment_prompted", candidate_requests + ) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("text-prompted segmentation gate must be an object") + minimum_mask_iou = float(gate.get("min_mask_iou", 0.7)) + minimum_match_rate = float(gate.get("min_mask_match_rate", 1.0)) + minimum_box_iou = float(gate.get("min_box_iou", 0.9)) + maximum_score_error = float(gate.get("max_score_abs_error", 0.05)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 + for value in ( + minimum_mask_iou, + minimum_match_rate, + minimum_box_iou, + maximum_score_error, + minimum_sample_rate, + ) + ): + raise QualificationError("text-prompted segmentation gates must be in [0, 1]") + + rows = [] + total_matches = total_masks = 0 + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate = _instance_masks(candidate_sample, "candidate") + reference = _instance_masks(reference_sample, "reference") + matches = _match_instances(candidate, reference) + denominator = max(len(candidate[2]), len(reference[2]), 1) + match_rate = len(matches) / denominator + minimum_sample_mask_iou = min((match["mask_iou"] for match in matches), default=0.0) + minimum_sample_box_iou = min((match["box_iou"] for match in matches), default=0.0) + maximum_sample_score_error = max( + (match["score_abs_error"] for match in matches), default=1.0 + ) + passed = ( + match_rate >= minimum_match_rate + and minimum_sample_mask_iou >= minimum_mask_iou + and minimum_sample_box_iou >= minimum_box_iou + and maximum_sample_score_error <= maximum_score_error + ) + rows.append( + { + "sample_id": sample["sample_id"], + "prompt": sample["label_name"], + "passed": passed, + "candidate_masks": len(candidate[2]), + "reference_masks": len(reference[2]), + "mask_match_rate": match_rate, + "min_mask_iou": minimum_sample_mask_iou, + "min_box_iou": minimum_sample_box_iou, + "max_score_abs_error": maximum_sample_score_error, + } + ) + total_matches += len(matches) + total_masks += denominator + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "mask_match_rate": total_matches / total_masks, + "min_mask_iou": min(row["min_mask_iou"] for row in rows), + "min_box_iou": min(row["min_box_iou"] for row in rows), + "max_score_abs_error": max(row["max_score_abs_error"] for row in rows), + }, + "gate": { + "min_mask_iou": minimum_mask_iou, + "min_mask_match_rate": minimum_match_rate, + "min_box_iou": minimum_box_iou, + "max_score_abs_error": maximum_score_error, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _normalized_answer(value: Any) -> str: + return " ".join(str(value or "").casefold().split()).strip(".,!?;:'\"") + + +def _normalized_edit_distance(left: str, right: str) -> float: + if left == right: + return 0.0 + if not left or not right: + return 1.0 + previous = list(range(len(right) + 1)) + for row, left_character in enumerate(left, start=1): + current = [row] + for column, right_character in enumerate(right, start=1): + current.append( + min( + current[column - 1] + 1, + previous[column] + 1, + previous[column - 1] + (left_character != right_character), + ) + ) + previous = current + return previous[-1] / max(len(left), len(right)) + + +def _vision_language_text_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "vision-language") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("vision-language request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {**dict(request), "image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("vision-language gate must be an object") + maximum_distance = float(gate.get("max_normalized_edit_distance", 0.15)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= maximum_distance <= 1.0 or not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("vision-language gates must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_text = _normalized_answer(candidate_sample.get("text")) + reference_text = _normalized_answer(reference_sample.get("text")) + distance = _normalized_edit_distance(candidate_text, reference_text) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": bool(candidate_text) and distance <= maximum_distance, + "candidate_text": candidate_text, + "reference_text": reference_text, + "normalized_edit_distance": distance, + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "max_normalized_edit_distance": max(row["normalized_edit_distance"] for row in rows), + }, + "gate": { + "max_normalized_edit_distance": maximum_distance, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _ocr_samples( + dataset: Dataset, + definition: Mapping[str, Any], + sample_limit: int, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("OCR dataset is not valid JSON") from error + samples = payload.get("samples") if isinstance(payload, Mapping) else None + if not isinstance(samples, list) or len(samples) < sample_limit: + raise QualificationError(f"OCR dataset requires at least {sample_limit} samples") + root = dataset.path.parent.resolve() + selected_rows, selection = _select_dataset_rows(samples, definition, sample_limit, "OCR") + selected = [] + for index, sample in enumerate(selected_rows): + if not isinstance(sample, Mapping): + raise QualificationError(f"OCR sample {index} must be an object") + media = sample.get("media") + image_value = None + if isinstance(media, list): + for item in media: + if isinstance(item, Mapping) and item.get("type") == "image": + image_value = item.get("path") + break + prompt = sample.get("question") + answer = sample.get("answer") + if not isinstance(image_value, str) or not image_value: + raise QualificationError(f"OCR sample {index} has no image") + if not isinstance(prompt, str) or not prompt.strip(): + raise QualificationError(f"OCR sample {index} has no question") + if not isinstance(answer, Mapping): + raise QualificationError(f"OCR sample {index} has no answer") + aliases = answer.get("aliases", []) + primary = answer.get("primary") + if not isinstance(aliases, list) or not all( + isinstance(value, str) and value.strip() for value in aliases + ): + raise QualificationError(f"OCR sample {index} has invalid answer aliases") + answers = [str(value).strip() for value in aliases] + if isinstance(primary, str) and primary.strip() and primary.strip() not in answers: + answers.insert(0, primary.strip()) + if not answers: + raise QualificationError(f"OCR sample {index} has no usable answer") + image = (root / image_value).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError(f"OCR sample {index} image is unavailable: {image}") + selected.append( + { + "sample_id": str(sample.get("id") or f"sample-{index}"), + "image_path": str(image), + "prompt": prompt.strip(), + "answers": answers, + } + ) + return selected, selection + + +def _ocr_gold_match(text: str, answers: Sequence[str]) -> bool: + normalized = _normalized_answer(text).strip(".,!?;:'\"") + return any(normalized == _normalized_answer(answer).strip(".,!?;:'\"") for answer in answers) + + +def _ocr_text_parity( + case: QualificationCase, + context: RuntimeContext, + definition: Mapping[str, Any], + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected, selection = _ocr_samples(dataset, definition, sample_limit) + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("OCR request must be an object") + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + **dict(request), + "image_path": sample["image_path"], + "prompt": sample["prompt"], + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("OCR gate must be an object") + maximum_distance = float(gate.get("max_normalized_edit_distance", 0.5)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not 0.0 <= maximum_distance <= 1.0 or not 0.0 <= minimum_sample_rate <= 1.0: + raise QualificationError("OCR gates must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_text = _normalized_answer(candidate_sample.get("text")) + reference_text = _normalized_answer(reference_sample.get("text")) + distance = _normalized_edit_distance(candidate_text, reference_text) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": bool(candidate_text) and distance <= maximum_distance, + "candidate_text": candidate_text, + "reference_text": reference_text, + "normalized_edit_distance": distance, + "candidate_gold_match": _ocr_gold_match(candidate_text, sample["answers"]), + "reference_gold_match": _ocr_gold_match(reference_text, sample["answers"]), + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + candidate_gold_rate = sum(bool(row["candidate_gold_match"]) for row in rows) / len(rows) + reference_gold_rate = sum(bool(row["reference_gold_match"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if sample_pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": {**dataset.receipt(), "selection": selection}, + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "max_normalized_edit_distance": max(row["normalized_edit_distance"] for row in rows), + "candidate_normalized_gold_match_rate": candidate_gold_rate, + "reference_normalized_gold_match_rate": reference_gold_rate, + }, + "gate": { + "max_normalized_edit_distance": maximum_distance, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +_LOCALIZATION_GROUP = re.compile(r"<(box|point)>(.*?)", re.DOTALL) + + +def _localization_values(text: str) -> tuple[str, tuple[tuple[float, ...], ...]]: + if "" not in text or "" not in text: + raise QualificationError("localization output omitted its reference tag") + groups = _LOCALIZATION_GROUP.findall(text) + if not groups: + raise QualificationError("localization output omitted box or point coordinates") + kind = groups[0][0] + values = [] + for current, raw in groups: + coordinates = tuple(float(value) for value in re.findall(r"-?\d+(?:\.\d+)?", raw)) + expected_size = 4 if current == "box" else 2 + if current != kind or len(coordinates) != expected_size: + raise QualificationError("localization output mixes coordinate kinds or ranks") + if not all(math.isfinite(value) and 0.0 <= value <= 1000.0 for value in coordinates): + raise QualificationError("localization coordinates must be finite values in [0, 1000]") + if current == "box" and ( + coordinates[2] <= coordinates[0] or coordinates[3] <= coordinates[1] + ): + raise QualificationError("localization box coordinates are not ordered xyxy") + values.append(coordinates) + return kind, tuple(values) + + +def _localization_alignment( + candidate: tuple[tuple[float, ...], ...], + reference: tuple[tuple[float, ...], ...], + kind: str, +) -> float: + if len(candidate) != len(reference): + return 0.0 if kind == "box" else math.inf + if kind == "box": + return max( + min(_box_iou(left, right) for left, right in zip(candidate, ordering, strict=True)) + for ordering in permutations(reference) + ) + return min( + max(math.dist(left, right) for left, right in zip(candidate, ordering, strict=True)) + for ordering in permutations(reference) + ) + + +def _localization_text_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "localization") + if any("label_name" not in sample for sample in selected): + raise QualificationError("localization dataset samples require label_name") + expected = _family_image_reference(case, context, output, selected) + request = configured.get("request", {}) + if not isinstance(request, Mapping): + raise QualificationError("localization request must be an object") + prompt_template = request.get("prompt_template") + if not isinstance(prompt_template, str) or "{label}" not in prompt_template: + raise QualificationError("localization request.prompt_template must contain {label}") + controls = {name: value for name, value in request.items() if name != "prompt_template"} + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": { + **controls, + "image_path": sample["image_path"], + "prompt": prompt_template.format(label=sample["label_name"]), + }, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("localization gate must be an object") + minimum_box_iou = float(gate.get("min_localization_box_iou", 0.9)) + maximum_point_distance = float(gate.get("max_localization_point_distance", 10.0)) + maximum_text_distance = float(gate.get("max_normalized_edit_distance", 0.5)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if ( + not all( + 0.0 <= value <= 1.0 + for value in (minimum_box_iou, maximum_text_distance, minimum_sample_rate) + ) + or maximum_point_distance < 0.0 + or not math.isfinite(maximum_point_distance) + ): + raise QualificationError("localization gates are outside their valid ranges") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + candidate_text = _normalized_answer(candidate_sample.get("text")) + reference_text = _normalized_answer(reference_sample.get("text")) + reference_kind, reference_values = _localization_values(reference_text) + try: + candidate_kind, candidate_values = _localization_values(candidate_text) + except QualificationError: + candidate_kind = "invalid" + candidate_values = () + same_contract = candidate_kind == reference_kind and bool(candidate_values) + alignment = ( + _localization_alignment(candidate_values, reference_values, candidate_kind) + if same_contract + else 0.0 + ) + text_distance = _normalized_edit_distance(candidate_text, reference_text) + localization_passed = ( + alignment >= minimum_box_iou + if candidate_kind == "box" + else alignment <= maximum_point_distance + if candidate_kind == "point" + else False + ) + rows.append( + { + "sample_id": sample["sample_id"], + "passed": localization_passed and text_distance <= maximum_text_distance, + "kind": candidate_kind, + "candidate_count": len(candidate_values), + "reference_count": len(reference_values), + "localization_alignment": alignment, + "normalized_edit_distance": text_distance, + "candidate_text": candidate_text, + "reference_text": reference_text, + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if sample_pass_rate >= minimum_sample_rate else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "max_normalized_edit_distance": max(row["normalized_edit_distance"] for row in rows), + }, + "gate": { + "min_localization_box_iou": minimum_box_iou, + "max_localization_point_distance": maximum_point_distance, + "max_normalized_edit_distance": maximum_text_distance, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _semantic_segmentation_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + selected = _image_parity_samples(dataset, sample_limit, "semantic-segmentation") + expected = _family_image_reference(case, context, output, selected) + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "segment", candidate_requests) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("semantic-segmentation gate must be an object") + minimum_pixel_accuracy = float(gate.get("min_pixel_accuracy", 0.99)) + minimum_mean_iou = float(gate.get("min_mean_iou", 0.94)) + minimum_sample_rate = float(gate.get("min_sample_pass_rate", 1.0)) + if not all( + 0.0 <= value <= 1.0 + for value in (minimum_pixel_accuracy, minimum_mean_iou, minimum_sample_rate) + ): + raise QualificationError("semantic-segmentation gates must be in [0, 1]") + + rows = [] + for sample, candidate_sample, reference_sample in zip(selected, actual, expected, strict=True): + left_height, left_width, left = _semantic_mask(candidate_sample, "candidate") + right_height, right_width, right = _semantic_mask(reference_sample, "reference") + if (left_height, left_width) != (right_height, right_width): + pixel_accuracy = mean_iou = 0.0 + else: + pixel_accuracy = sum(a == b for a, b in zip(left, right, strict=True)) / len(left) + class_ious = [] + for class_id in sorted(set(left) | set(right)): + intersection = sum(a == class_id and b == class_id for a, b in zip(left, right)) + union = sum(a == class_id or b == class_id for a, b in zip(left, right)) + if union: + class_ious.append(intersection / union) + mean_iou = math.fsum(class_ious) / len(class_ious) if class_ious else 0.0 + rows.append( + { + "sample_id": sample["sample_id"], + "passed": pixel_accuracy >= minimum_pixel_accuracy and mean_iou >= minimum_mean_iou, + "pixel_accuracy": pixel_accuracy, + "mean_iou": mean_iou, + "candidate_shape": [left_height, left_width], + "reference_shape": [right_height, right_width], + } + ) + sample_pass_rate = sum(bool(row["passed"]) for row in rows) / len(rows) + passed = sample_pass_rate >= minimum_sample_rate + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "sample_pass_rate": sample_pass_rate, + "mean_pixel_accuracy": math.fsum(row["pixel_accuracy"] for row in rows) / len(rows), + "mean_iou": math.fsum(row["mean_iou"] for row in rows) / len(rows), + "min_pixel_accuracy": min(row["pixel_accuracy"] for row in rows), + "min_mean_iou": min(row["mean_iou"] for row in rows), + }, + "gate": { + "min_pixel_accuracy": minimum_pixel_accuracy, + "min_mean_iou": minimum_mean_iou, + "min_sample_pass_rate": minimum_sample_rate, + }, + "samples": rows, + } + + +def _image_classification_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + try: + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise QualificationError("image-classification dataset is not valid JSON") from error + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError( + f"image-classification dataset requires at least {sample_limit} requests" + ) + + selected: list[dict[str, Any]] = [] + dataset_root = dataset.path.parent.resolve() + for index, request in enumerate(requests[:sample_limit]): + if not isinstance(request, Mapping): + raise QualificationError(f"image-classification request {index} must be an object") + relative = request.get("image") + label = request.get("label") + if not isinstance(relative, str) or not relative: + raise QualificationError(f"image-classification request {index} has no image") + if isinstance(label, bool) or not isinstance(label, int) or label < 0: + raise QualificationError(f"image-classification request {index} has no valid label") + image = (dataset_root / relative).resolve() + if dataset_root not in image.parents or not image.is_file(): + raise QualificationError( + f"image-classification request {index} image is unavailable: {image}" + ) + selected.append( + { + "sample_id": str(request.get("id") or f"sample-{index}"), + "image_path": str(image), + "label": label, + } + ) + + reference = configured.get("reference", {}) + if not isinstance(reference, Mapping): + raise QualificationError("image-classification reference must be an object") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "batch_size": int(reference.get("batch_size", 16)), + "samples": selected, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + runner = ( + context.repository / "qualification_tests/benchmark_qualification/references/timm_image_classification.py" + ) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=3600, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"TIMM Accuracy reference failed; see {output}") + reference_payload = json.loads(reference_path.read_text(encoding="utf-8")) + expected = reference_payload.get("samples") + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("TIMM Accuracy reference returned an invalid sample set") + + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"image_path": sample["image_path"]}, + } + for sample in selected + ] + actual, bundle = _candidate_outputs(case, context, output, "classify", candidate_requests) + rows = [] + for sample, reference_sample, candidate_sample in zip(selected, expected, actual, strict=True): + reference_class = reference_sample.get("top_class") + candidate_class = candidate_sample.get("top_class") + if any( + isinstance(value, bool) or not isinstance(value, int) + for value in (reference_class, candidate_class) + ): + raise QualificationError("classification output omitted an integer top_class") + rows.append( + { + "sample_id": sample["sample_id"], + "label": sample["label"], + "reference_top_class": reference_class, + "candidate_top_class": candidate_class, + "reference_correct": reference_class == sample["label"], + "candidate_correct": candidate_class == sample["label"], + "passed": candidate_class == reference_class, + } + ) + + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("image-classification gate must be an object") + minimum_agreement = float(gate.get("min_top1_agreement", 0.98)) + maximum_drop = float(gate.get("max_top1_accuracy_drop_from_hf", 0.01)) + if not 0.0 <= minimum_agreement <= 1.0: + raise QualificationError("min_top1_agreement must be in [0, 1]") + if maximum_drop < 0.0 or not math.isfinite(maximum_drop): + raise QualificationError("max_top1_accuracy_drop_from_hf must be finite and nonnegative") + agreement = sum(row["passed"] for row in rows) / len(rows) + reference_accuracy = sum(row["reference_correct"] for row in rows) / len(rows) + candidate_accuracy = sum(row["candidate_correct"] for row in rows) / len(rows) + passed = ( + agreement >= minimum_agreement and candidate_accuracy >= reference_accuracy - maximum_drop + ) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "samples": len(rows), + "top1_agreement": agreement, + "reference_top1_accuracy": reference_accuracy, + "candidate_top1_accuracy": candidate_accuracy, + "top1_accuracy_drop_from_hf": reference_accuracy - candidate_accuracy, + }, + "gate": { + "min_top1_agreement": minimum_agreement, + "max_top1_accuracy_drop_from_hf": maximum_drop, + }, + "samples": rows, + } + + +def _speech_transcription_parity( + case: QualificationCase, + context: RuntimeContext, + definition: Mapping[str, Any], + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") + payload = json.loads(dataset.path.read_text(encoding="utf-8")) + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or len(requests) < sample_limit: + raise QualificationError(f"speech dataset requires at least {sample_limit} requests") + + selected_requests, selection = _select_dataset_rows( + requests, definition, sample_limit, "speech" + ) + selected = [] + data_root = context.data_root.resolve() + for index, request in enumerate(selected_requests): + if not isinstance(request, Mapping): + raise QualificationError(f"speech request {index} must be an object") + audio_value = _speech_audio(request) + audio = (data_root / audio_value).resolve() + if data_root not in audio.parents or not audio.is_file(): + fallback = (dataset.path.parent / audio_value).resolve() + if data_root not in fallback.parents or not fallback.is_file(): + raise QualificationError(f"speech request {index} audio is unavailable: {audio}") + audio = fallback + gold = request.get("reference") + if not isinstance(gold, str) or not gold.strip(): + raise QualificationError(f"speech request {index} has no reference transcript") + selected.append( + { + "sample_id": str(request.get("id") or f"sample-{index}"), + "audio_path": str(audio), + "gold_text": gold, + } + ) + + reference = configured.get("reference") + candidate_request = configured.get("request", {}) + if not isinstance(reference, Mapping) or not isinstance(candidate_request, Mapping): + raise QualificationError("speech Accuracy reference and request must be objects") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("speech Accuracy reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"speech reference runner is not family-owned: {runner}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "max_new_tokens": int(candidate_request.get("max_new_tokens", 128)), + "language": candidate_request.get("language"), + "samples": selected, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"speech Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(selected): + raise QualificationError("speech reference returned an invalid sample set") + + candidate_requests = [] + for sample in expected: + if not isinstance(sample, Mapping): + raise QualificationError("speech reference sample must be an object") + audio_path = sample.get("audio_path") + if not isinstance(audio_path, str) or not Path(audio_path).is_file(): + raise QualificationError("speech reference did not produce candidate WAV audio") + candidate_requests.append( + { + "sample_id": str(sample.get("sample_id", "")), + "request": {**dict(candidate_request), "audio_path": audio_path}, + } + ) + actual, bundle = _candidate_outputs(case, context, output, "transcribe", candidate_requests) + + rows = [] + gold_reference_counts = [0, 0] + gold_candidate_counts = [0, 0] + parity_counts = [0, 0] + for source, reference_sample, candidate_sample in zip(selected, expected, actual, strict=True): + reference_text = reference_sample.get("text") + candidate_text = candidate_sample.get("text") + if not isinstance(reference_text, str) or not reference_text.strip(): + raise QualificationError("speech reference returned an empty transcript") + if not isinstance(candidate_text, str): + raise QualificationError("speech candidate omitted its transcript") + reference_gold = _word_error_counts(source["gold_text"], reference_text) + candidate_gold = _word_error_counts(source["gold_text"], candidate_text) + candidate_reference = _word_error_counts(reference_text, candidate_text) + for totals, counts in ( + (gold_reference_counts, reference_gold), + (gold_candidate_counts, candidate_gold), + (parity_counts, candidate_reference), + ): + totals[0] += counts[0] + totals[1] += counts[1] + rows.append( + { + "sample_id": source["sample_id"], + "reference_text": reference_text, + "candidate_text": candidate_text, + "gold_text": source["gold_text"], + "reference_wer": _rate(reference_gold), + "candidate_wer": _rate(candidate_gold), + "wer_to_reference": _rate(candidate_reference), + } + ) + + reference_wer = _rate(gold_reference_counts) + candidate_wer = _rate(gold_candidate_counts) + parity_wer = _rate(parity_counts) + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("speech Accuracy gate must be an object") + parity_limit = float(gate.get("max_wer_to_reference", 0.1)) + increase_limit = float(gate.get("max_wer_increase_from_reference", 0.02)) + if any(not math.isfinite(value) or value < 0.0 for value in (parity_limit, increase_limit)): + raise QualificationError("speech WER gates must be finite and nonnegative") + increase = candidate_wer - reference_wer + passed = parity_wer <= parity_limit and increase <= increase_limit + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": {**dataset.receipt(), "selection": selection}, + "metrics": { + "samples": len(rows), + "reference_wer": reference_wer, + "candidate_wer": candidate_wer, + "wer_increase_from_reference": increase, + "wer_to_reference": parity_wer, + }, + "gate": { + "max_wer_to_reference": parity_limit, + "max_wer_increase_from_reference": increase_limit, + }, + "samples": rows, + } + + +def _image_feature_knn_parity( + case: QualificationCase, + context: RuntimeContext, + dataset: Dataset, + output: Path, +) -> dict[str, Any]: + configured = case.values + bank_per_class = _positive_int( + configured.get("bank_samples_per_class"), "accuracy.bank_samples_per_class" + ) + query_per_class = _positive_int( + configured.get("query_samples_per_class"), "accuracy.query_samples_per_class" + ) + bank, queries = _image_feature_samples(dataset.path, bank_per_class, query_per_class) + samples = [*bank, *queries] + + reference = configured.get("reference") + if not isinstance(reference, Mapping): + raise QualificationError("image-feature Accuracy reference must be an object") + runner_value = reference.get("command") + if not isinstance(runner_value, str) or not runner_value: + raise QualificationError("image-feature Accuracy reference.command must be set") + runner = (case.source.parent / runner_value).resolve() + family_root = case.source.parents[2].resolve() + if family_root not in runner.parents or not runner.is_file(): + raise QualificationError(f"image-feature reference runner is not family-owned: {runner}") + reference_request = { + "model": str(case.candidate["checkpoint"]), + "revision": case.candidate.get("revision"), + "precision": str(reference.get("precision", "fp32")), + "batch_size": int(reference.get("batch_size", 16)), + "samples": samples, + } + request_path = output / "reference-request.json" + reference_path = output / "reference.json" + _json(request_path, reference_request) + completed = run_command( + [ + str(reference_python(case, context)), + str(runner), + "--request", + str(request_path), + "--output", + str(reference_path), + ], + output, + "reference", + timeout=7200, + verbose=context.verbose, + ) + if completed.returncode != 0: + raise QualificationError(f"image-feature Accuracy reference failed; see {output}") + expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") + if not isinstance(expected, list) or len(expected) != len(samples): + raise QualificationError("image-feature reference returned an invalid sample set") + + candidate_requests = [ + { + "sample_id": sample["sample_id"], + "request": {"image_path": sample["image_path"]}, + } + for sample in samples + ] + actual, bundle = _candidate_outputs( + case, context, output, "extract_features", candidate_requests + ) + reference_vectors = [ + _finite_vector(sample.get("pooler_output", []), "HF image feature") for sample in expected + ] + candidate_vectors = [ + _finite_vector(sample.get("pooler_output", []), "TRTMC image feature") for sample in actual + ] + if any(len(left) != len(right) for left, right in zip(reference_vectors, candidate_vectors)): + raise QualificationError("image-feature reference and candidate dimensions differ") + + gate = configured.get("gate", {}) + if not isinstance(gate, Mapping): + raise QualificationError("image-feature Accuracy gate must be an object") + minimum_cosine = float(gate.get("min_pooler_cosine", 0.999)) + minimum_rate = float(gate.get("min_vector_pass_rate", 1.0)) + minimum_agreement = float(gate.get("min_knn_top1_agreement", 0.98)) + maximum_drop = float(gate.get("max_knn_accuracy_drop_from_hf", 0.02)) + if not -1.0 <= minimum_cosine <= 1.0: + raise QualificationError("min_pooler_cosine must be in [-1, 1]") + if not 0.0 <= minimum_rate <= 1.0 or not 0.0 <= minimum_agreement <= 1.0: + raise QualificationError("image-feature pass rates must be in [0, 1]") + if maximum_drop < 0.0 or not math.isfinite(maximum_drop): + raise QualificationError("max_knn_accuracy_drop_from_hf must be finite and nonnegative") + + vector_rows = [] + cosines = [] + for sample, left, right in zip(samples, reference_vectors, candidate_vectors, strict=True): + cosine = _vector_cosine(left, right) + cosines.append(cosine) + vector_rows.append( + { + "sample_id": sample["sample_id"], + "split": sample["split"], + "label": sample["label"], + "pooler_cosine": cosine, + "passed": cosine >= minimum_cosine, + } + ) + + bank_count = len(bank) + knn_k = min(_positive_int(configured.get("knn_k", 10), "accuracy.knn_k"), bank_count) + temperature = float(configured.get("knn_temperature", 0.07)) + if not math.isfinite(temperature) or temperature <= 0.0: + raise QualificationError("accuracy.knn_temperature must be finite and positive") + labels = [sample["label"] for sample in bank] + reference_predictions = _knn_predictions( + reference_vectors[:bank_count], labels, reference_vectors[bank_count:], knn_k, temperature + ) + candidate_predictions = _knn_predictions( + candidate_vectors[:bank_count], labels, candidate_vectors[bank_count:], knn_k, temperature + ) + query_rows = [] + for sample, expected_label, candidate_label in zip( + queries, reference_predictions, candidate_predictions, strict=True + ): + query_rows.append( + { + "sample_id": sample["sample_id"], + "label": sample["label"], + "reference_top1": expected_label, + "candidate_top1": candidate_label, + "reference_correct": expected_label == sample["label"], + "candidate_correct": candidate_label == sample["label"], + "passed": candidate_label == expected_label, + } + ) + vector_pass_rate = sum(row["passed"] for row in vector_rows) / len(vector_rows) + agreement = sum(row["passed"] for row in query_rows) / len(query_rows) + reference_accuracy = sum(row["reference_correct"] for row in query_rows) / len(query_rows) + candidate_accuracy = sum(row["candidate_correct"] for row in query_rows) / len(query_rows) + accuracy_drop = reference_accuracy - candidate_accuracy + passed = ( + vector_pass_rate >= minimum_rate + and agreement >= minimum_agreement + and accuracy_drop <= maximum_drop + ) + return { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "accuracy", + "status": "passed" if passed else "failed", + "model": case.model, + "benchmark": case.benchmark, + "bundle": str(bundle), + "dataset": dataset.receipt(), + "metrics": { + "bank_samples": bank_count, + "query_samples": len(queries), + "vector_pass_rate": vector_pass_rate, + "mean_pooler_cosine": math.fsum(cosines) / len(cosines), + "min_pooler_cosine": min(cosines), + "knn_top1_agreement": agreement, + "reference_knn_top1_accuracy": reference_accuracy, + "candidate_knn_top1_accuracy": candidate_accuracy, + "knn_accuracy_drop_from_hf": accuracy_drop, + }, + "gate": { + "min_pooler_cosine": minimum_cosine, + "min_vector_pass_rate": minimum_rate, + "min_knn_top1_agreement": minimum_agreement, + "max_knn_accuracy_drop_from_hf": maximum_drop, + }, + "samples": vector_rows, + "queries": query_rows, + } + + +def _image_feature_samples( + dataset_path: Path, bank_per_class: int, query_per_class: int +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + payload = json.loads(dataset_path.read_text(encoding="utf-8")) + requests = payload.get("requests") if isinstance(payload, Mapping) else None + if not isinstance(requests, list) or not requests: + raise QualificationError("image-feature dataset must contain requests") + root = dataset_path.parent.resolve() + bank_path: Path | None = None + query_paths: list[Path] = [] + for request in requests: + inputs = request.get("inputs") if isinstance(request, Mapping) else None + if not isinstance(inputs, Mapping): + raise QualificationError("image-feature dataset request must contain inputs") + configured_bank = _dataset_child(root, inputs.get("bank_manifest"), "bank manifest") + if bank_path is not None and bank_path != configured_bank: + raise QualificationError("image-feature dataset requests use different banks") + bank_path = configured_bank + query_paths.append(_dataset_child(root, inputs.get("query_manifest"), "query manifest")) + assert bank_path is not None + bank = _labeled_image_manifest(bank_path, root, "bank") + queries = [ + sample + for query_path in query_paths + for sample in _labeled_image_manifest(query_path, root, "query") + ] + return ( + _take_per_class(bank, bank_per_class, "bank"), + _take_per_class(queries, query_per_class, "query"), + ) + + +def _dataset_child(root: Path, value: Any, label: str) -> Path: + if not isinstance(value, str) or not value: + raise QualificationError(f"image-feature dataset omitted its {label}") + path = (root / value).resolve() + if root not in path.parents or not path.is_file(): + raise QualificationError(f"image-feature {label} is unavailable: {path}") + return path + + +def _labeled_image_manifest(path: Path, root: Path, split: str) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + samples = payload.get("samples") if isinstance(payload, Mapping) else None + if not isinstance(samples, list) or not samples: + raise QualificationError(f"image-feature {split} manifest has no samples") + result = [] + for index, sample in enumerate(samples): + relative = sample.get("image") if isinstance(sample, Mapping) else None + label = sample.get("label") if isinstance(sample, Mapping) else None + if not isinstance(relative, str) or isinstance(label, bool) or not isinstance(label, int): + raise QualificationError(f"image-feature {split} sample {index} is invalid") + image = (path.parent / relative).resolve() + if root not in image.parents or not image.is_file(): + raise QualificationError(f"image-feature {split} image is unavailable: {image}") + result.append( + { + "sample_id": f"{split}-{int(sample.get('source_index', index)):06d}", + "image_path": str(image), + "label": label, + "split": split, + } + ) + return result + + +def _take_per_class( + samples: Sequence[Mapping[str, Any]], count: int, split: str +) -> list[dict[str, Any]]: + classes = sorted({int(sample["label"]) for sample in samples}) + selected = [] + for label in classes: + matches = [dict(sample) for sample in samples if sample["label"] == label][:count] + if len(matches) != count: + raise QualificationError( + f"image-feature {split} class {label} has {len(matches)} samples; {count} required" + ) + selected.extend(matches) + return selected + + +def _knn_predictions( + bank: Sequence[Sequence[float]], + labels: Sequence[int], + queries: Sequence[Sequence[float]], + k: int, + temperature: float, +) -> list[int]: + predictions = [] + for query in queries: + neighbors = sorted( + ( + (_vector_cosine(query, vector), label) + for vector, label in zip(bank, labels, strict=True) + ), + reverse=True, + )[:k] + scores: dict[int, float] = {} + for similarity, label in neighbors: + scores[label] = scores.get(label, 0.0) + math.exp(similarity / temperature) + predictions.append(max(scores, key=lambda label: (scores[label], -label))) + return predictions + + +def _speech_audio(request: Mapping[str, Any]) -> str: + messages = request.get("messages") + if not isinstance(messages, list): + raise QualificationError("speech request has no messages") + for message in messages: + if not isinstance(message, Mapping): + continue + content = message.get("content") + if not isinstance(content, list): + continue + for item in content: + if isinstance(item, Mapping) and item.get("type") == "audio": + audio = item.get("audio") + if isinstance(audio, str) and audio: + return audio + raise QualificationError("speech request has no audio content") + + +def _word_error_counts(reference: str, hypothesis: str) -> list[int]: + def words(value: str) -> list[str]: + return [ + normalized + for word in value.split() + if (normalized := re.sub(r"^[^\w]+|[^\w]+$", "", word).casefold()) + ] + + left = words(reference) + right = words(hypothesis) + previous = list(range(len(right) + 1)) + for row, left_word in enumerate(left, start=1): + current = [row] + for column, right_word in enumerate(right, start=1): + current.append( + min( + current[-1] + 1, + previous[column] + 1, + previous[column - 1] + (left_word != right_word), + ) + ) + previous = current + return [previous[-1], len(left)] + + +def _rate(counts: Sequence[int]) -> float: + return float(counts[0]) / max(int(counts[1]), 1) + + +def _etth1_windows( + path: Path, + definition: Mapping[str, Any], + configured: Mapping[str, Any], + count: int, +) -> list[dict[str, Any]]: + with path.open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + columns = configured.get("columns", ["OT"]) + if not isinstance(columns, list) or not columns: + raise QualificationError("ETTh1 window.columns must be a non-empty list") + context = int(configured.get("context_length", 512)) + prediction = int(configured.get("prediction_length", 64)) + start = int(configured.get("test_target_start", 11520)) + end = int(configured.get("test_end", 14400)) + stride = int(configured.get("stride", 24)) + starts = list(range(start - context, end - context - prediction + 1, stride)) + seed = int(definition.get("selection", {}).get("seed", 20260715)) + random.Random(seed).shuffle(starts) + if count < 1 or len(starts) < count or len(rows) < end: + raise QualificationError("ETTh1 dataset cannot satisfy the configured windows") + return [ + { + "sample_id": f"etth1-{index:04d}", + "past_values": [ + float(row[column]) + for row in rows[window_start : window_start + context] + for column in columns + ], + "frequency": int(configured.get("frequency", 0)), + } + for index, window_start in enumerate(starts[:count]) + ] + + +def _prompt(request: Mapping[str, Any]) -> str: + messages = request.get("messages") + if isinstance(messages, list): + for message in reversed(messages): + if ( + isinstance(message, Mapping) + and message.get("role") == "user" + and isinstance(message.get("content"), str) + and message["content"].strip() + ): + return str(message["content"]) + prompt = request.get("prompt") + if isinstance(prompt, str) and prompt.strip(): + return prompt + inputs = request.get("inputs") + if isinstance(inputs, Mapping): + prompt = inputs.get("prompt") + if isinstance(prompt, str) and prompt.strip(): + return prompt + raise QualificationError("text-generation request has neither a prompt nor a user message") + + +def _select_dataset_rows( + rows: Sequence[Any], + definition: Mapping[str, Any], + count: int, + label: str, +) -> tuple[list[Any], dict[str, Any]]: + selection = definition.get("selection", {}) + if not isinstance(selection, Mapping): + raise QualificationError(f"{label} dataset selection must be an object") + method = str(selection.get("method", "first")) + if method == "first": + if len(rows) < count: + raise QualificationError(f"{label} dataset requires at least {count} samples") + indices = list(range(count)) + elif method == "fixed-indices": + configured = selection.get("indices") + if not isinstance(configured, list) or len(configured) < count: + raise QualificationError( + f"{label} fixed dataset selection requires at least {count} indices" + ) + if any( + isinstance(index, bool) or not isinstance(index, int) or index < 0 + for index in configured + ): + raise QualificationError(f"{label} fixed dataset indices must be nonnegative integers") + if len(set(configured)) != len(configured): + raise QualificationError(f"{label} fixed dataset indices must be unique") + indices = configured[:count] + if any(index >= len(rows) for index in indices): + raise QualificationError(f"{label} fixed dataset index is out of range") + else: + raise QualificationError(f"unsupported {label} dataset selection method {method!r}") + return [rows[index] for index in indices], {"method": method, "indices": indices} + + +def _positive_int(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise QualificationError(f"{name} must be a positive integer") + return value + + +def _json(path: Path, value: Any) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml new file mode 100644 index 0000000000..7ba248d3a7 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml new file mode 100644 index 0000000000..4ab9b70fec --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml b/qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml new file mode 100644 index 0000000000..bb5cbc4915 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: dinov3-beans-knn-v1 + version: AI-Lab-Makerere/beans@27aa014ce09b193e1a6f58112d4a66e0eddb69c5 + sha256: 77ea4c7f6204e212d92f4b01b8c47a8d1f2db59b7448f266c5b3a9ebc418b0f3 + source: + mode: manual + path: dinov3-beans-knn-v1/dataset.json + instructions: https://huggingface.co/datasets/AI-Lab-Makerere/beans + license: MIT +selection: + method: first-per-class +input: + adapter: image_path +metric: + name: image_feature_knn_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml b/qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml new file mode 100644 index 0000000000..3d692db04a --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: coco2017-object-detection + version: val2017-category-balanced-100 + sha256: 5fca4f7e831e99cc721fc9340bf76dc48086548d4fb4a7c84512a18b91edae20 + source: + mode: manual + path: COCO2017_object_detection/coco2017_object_detection.json + instructions: tools/prepare_coco_detection_dataset.py + license: COCO images retain their source licenses; annotations are CC BY 4.0 +selection: + method: fixed-manifest-order +input: + adapter: image_path +metric: + name: coco_object_detection_accuracy diff --git a/qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/tools/benchmark_qualification/benchmarks/text_generation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/encoder_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/text_generation_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/encoder_performance.yaml diff --git a/tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml rename to qualification_tests/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml diff --git a/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml b/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml new file mode 100644 index 0000000000..dbe017cf61 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: flores200-en-fr-seq2seq + version: pre1093-flores200-en-fr-seq2seq-json + sha256: a85db4d8bb5f03e216432d9d80a6c196ab3cae79257de2d486d7339d55a50c4c + source: + mode: manual + path: flores200-en-fr/dataset.json + instructions: https://github.com/facebookresearch/flores + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: model_plugin_json +metric: + name: exact_token_ids diff --git a/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml b/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml new file mode 100644 index 0000000000..db842d2b6f --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: flores200-en-fr-riva + version: pre1093-flores200-en-fr-riva-json + sha256: a797a331fa208a48f6a168c3172e741795c92f70655726f20d8065665bd00534 + source: + mode: manual + path: FLORES200_en_fr/flores200_en_fr_task_eval.json + instructions: https://github.com/facebookresearch/flores + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml b/qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml similarity index 55% rename from tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml rename to qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml index 31cbfa36ad..875c9d5f43 100644 --- a/tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml +++ b/qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml @@ -4,17 +4,17 @@ # Internal reusable benchmark definition; it does not register a model. kind: accuracy dataset: - id: mmlu-five-shot - version: pre1093-mmlu-five-shot-json - sha256: f4f8c39761c86510888a860915e9ecbb73f418dc226c1368319630538859a470 + id: humaneval-continuation + version: pre1093-humaneval-json + sha256: 68eece17bfa17f4513fcd13b0919fd1a672bf58178224ed3cada1c26f6b97cc9 source: mode: manual - path: MMLU_five_shot/mmlu_dataset.json - instructions: https://github.com/hendrycks/test + path: HumanEval/humaneval_task_eval.json + instructions: https://github.com/openai/human-eval license: MIT selection: method: first input: - adapter: mmlu_continuation + adapter: text_generation_json metric: name: exact_token_ids diff --git a/tools/benchmark_qualification/benchmarks/time_series_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/image_classification_performance.yaml similarity index 100% rename from tools/benchmark_qualification/benchmarks/time_series_performance.yaml rename to qualification_tests/benchmark_qualification/benchmarks/image_classification_performance.yaml diff --git a/qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml new file mode 100644 index 0000000000..d35af76dc9 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: image_classification_top1_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml new file mode 100644 index 0000000000..78bf6c8ea8 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_label_prompt +metric: + name: localization_text_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml new file mode 100644 index 0000000000..ccc49d093a --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: prompted_segmentation_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml new file mode 100644 index 0000000000..1c418f5a1f --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: semantic_segmentation_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml new file mode 100644 index 0000000000..984a8f7d64 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_label_prompt +metric: + name: text_prompted_instance_segmentation_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml new file mode 100644 index 0000000000..944f02a17c --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: imagenette-validation + version: imagenette2-320-task-eval + sha256: 07cd93194d148b8b68d89ba537b7a43741810e02a22429578a60bb85aeab5ea3 + source: + mode: manual + path: Imagenette/imagenette2-320_task_eval.json + instructions: https://github.com/fastai/imagenette + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: vision_language_text_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml b/qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml new file mode 100644 index 0000000000..4e8fe3a79e --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: librispeech-clean-test + version: test-clean-task-eval + sha256: 45306828857c7e4174baa52dab089214a8b2956165dacad3eaec8d2aa1483b1f + source: + mode: manual + path: librispeech_clean_test/librispeech_clean_test.json + instructions: https://www.openslr.org/12 + license: CC-BY-4.0 +selection: + # Ten evenly distributed utterances from distinct speaker/chapter pairs keep + # the existing runtime without overfitting qualification to one speaker. + method: fixed-indices + indices: + - 131 # 1320-122612-0011 + - 393 # 3575-170457-0024 + - 655 # 8230-279154-0038 + - 917 # 5142-33396-0019 + - 1179 # 4446-2273-0021 + - 1441 # 8224-274384-0004 + - 1703 # 3729-6852-0043 + - 1965 # 1580-141083-0016 + - 2227 # 61-70968-0048 + - 2489 # 5683-32865-0006 +input: + adapter: audio_path +metric: + name: speech_transcription_wer_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml new file mode 100644 index 0000000000..4ab9b70fec --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml new file mode 100644 index 0000000000..d901770179 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: metric-geometry-test-image + version: family-test-image-v1 + sha256: b600ba0a7d131e831dcb3ff4140759b8c3d6bd319345cf5f3756df8c8f1bf5c2 + source: + mode: family + path: ../data/benchmark_geometry.json + license: Apache-2.0 +selection: + method: first +input: + adapter: image_path +metric: + name: metric_geometry_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml b/qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml new file mode 100644 index 0000000000..8ae0af902d --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: mmlu-five-shot + version: pre1093-mmlu-five-shot-json + sha256: f4f8c39761c86510888a860915e9ecbb73f418dc226c1368319630538859a470 + source: + mode: manual + path: MMLU_five_shot/mmlu_dataset.json + instructions: https://github.com/hendrycks/test + license: MIT +selection: + # Ten subjects keep the existing runtime while covering STEM, medicine, + # humanities, social science, and professional knowledge. + method: fixed-indices + indices: + - 50 # abstract_algebra + - 167 # anatomy + - 437 # business_ethics + - 1046 # college_computer_science + - 1863 # econometrics + - 5970 # high_school_world_history + - 6891 # machine_learning + - 11398 # professional_law + - 13504 # sociology + - 13956 # world_religions +input: + adapter: mmlu_continuation +metric: + name: exact_token_ids diff --git a/qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml b/qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml new file mode 100644 index 0000000000..4c7a94a107 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: newstest2019-en-ru + version: pre1093-newstest2019-en-ru-json + sha256: 27b30a1ab0f8a6e4aa3025c3381a5e610a46c1d03140a9b50312601bcfe38c35 + source: + mode: manual + path: WMT19_en_ru/newstest2019_en_ru_task_eval.json + instructions: https://www.statmt.org/wmt19/translation-task.html + license: dataset-specific +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml new file mode 100644 index 0000000000..32222af0ca --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: ocrbench-v2 + version: unified-task-eval-v1 + sha256: e62c2fbdfe5d02f11d6e48d37ae548c854a19dcb3985eaa7ba3ba51492d10a94 + source: + mode: manual + path: OCRBench_v2/unified/dataset.json + instructions: https://huggingface.co/datasets/ling99/OCRBench_v2 + license: MIT +selection: + # Ten task types keep the existing runtime while covering recognition, + # parsing, extraction, reasoning, tables, charts, formulas, and translation. + method: fixed-indices + indices: + - 150 # APP agent en / rico + - 2100 # chart parsing en / OneChart + - 3950 # document parsing cn / CDLA + - 7100 # fine-grained text recognition en / spotting_cocotext + - 4900 # formula recognition en / HME100K + - 5700 # full-page OCR cn / CDLA + - 1100 # key information extraction en / POIE + - 6300 # reasoning VQA en / mmsi + - 8000 # table parsing en / table_to_html + - 9800 # text translation cn / memes +input: + adapter: ocrbench_v2 +metric: + name: ocr_text_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml new file mode 100644 index 0000000000..68ca3fed63 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: lerobot-act-recorded-observation + version: episode-0-frame-0-v1 + sha256: 33eb2456e87abbe65566c186b21fade3a410a3d230fe0fee96009077ca70181a + source: + mode: family + path: ../data/benchmark_robot_control.json + license: Apache-2.0 +selection: + method: first +input: + adapter: image_state_paths +metric: + name: robot_action_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml new file mode 100644 index 0000000000..d12095f992 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: true +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml new file mode 100644 index 0000000000..76b25771e1 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: stereo-office-pair + version: family-office-pair-v1 + sha256: ec5cac38fb583e821b7e19498d04eccd7bf22cb70718fa0d051090d6cdf1e893 + source: + mode: family + path: ../data/benchmark_stereo.json + license: Apache-2.0 +selection: + method: first +input: + adapter: stereo_image_paths +metric: + name: stereo_disparity_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml new file mode 100644 index 0000000000..5f95a3d08b --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml new file mode 100644 index 0000000000..1e6c4e3cf2 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: stsbenchmark-test + version: pre1093-mteb-stsbenchmark-test-jsonl + sha256: 7c8927eef1ac4e256cae9768afc76232f79c39d19cb05d16bef92e11cecbe5bb + source: + mode: manual + path: STSBenchmark/stsbenchmark_test.jsonl + instructions: https://huggingface.co/datasets/mteb/stsbenchmark-sts + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: sts_pair_jsonl +metric: + name: embedding_vector_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml new file mode 100644 index 0000000000..387117abb7 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: stsbenchmark-test + version: pre1093-mteb-stsbenchmark-test-jsonl + sha256: 7c8927eef1ac4e256cae9768afc76232f79c39d19cb05d16bef92e11cecbe5bb + source: + mode: manual + path: STSBenchmark/stsbenchmark_test.jsonl + instructions: https://huggingface.co/datasets/mteb/stsbenchmark-sts + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: sts_reranking_jsonl +metric: + name: reranking_score_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml b/qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml new file mode 100644 index 0000000000..61affffefb --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Family profiles own the request and official reference. Generated tasks do +# not require an unrelated external dataset: the prompt/assets are the case. +kind: accuracy +input: + adapter: configured_request +metric: + name: task_output_parity diff --git a/qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml new file mode 100644 index 0000000000..70c8984fbf --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: public_operation_call_wall + input_preparation_included: true + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml new file mode 100644 index 0000000000..e6b7601d06 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-model-call-wall + input_preparation_included: false + asset_loading_included: false +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml b/qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml new file mode 100644 index 0000000000..d12095f992 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: performance +reference_timing: + timing_scope: task-pipeline-call-wall + input_preparation_included: true + asset_loading_included: true +stability: + samples: 10 + max_half_median_change_percent: 5.0 + median_band_percent: 5.0 + minimum_samples_within_band: 8 + retries: 1 diff --git a/qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml b/qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml new file mode 100644 index 0000000000..14e9c67898 --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: wikitext103-continuation + version: pre1093-wikitext103-json + sha256: 25580c3a6563a931d1feb5eb84a45056472a0ed22bfdb536b182d96a034af887 + source: + mode: manual + path: WikiText103/wikitext103_task_eval.json + instructions: https://blog.salesforceairesearch.com/the-wikitext-long-term-dependency-language-modeling-dataset/ + license: CC BY-SA 4.0 +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml b/qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml new file mode 100644 index 0000000000..3c18ae0d4f --- /dev/null +++ b/qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Internal reusable benchmark definition; it does not register a model. +kind: accuracy +dataset: + id: wmt14-en-de-t5 + version: pre1093-wmt14-en-de-t5-json + sha256: 7851c9e2ed3627bbc8b0420415eeb2360422418a8e31e8dd5726b666c450479a + source: + mode: manual + path: WMT14_de_en/wmt14_en_de_t5_task_eval.json + instructions: https://www.statmt.org/wmt14/translation-task.html + license: dataset-specific +selection: + method: first +input: + adapter: text_generation_json +metric: + name: exact_token_ids diff --git a/tools/benchmark_qualification/catalog.py b/qualification_tests/benchmark_qualification/catalog.py similarity index 55% rename from tools/benchmark_qualification/catalog.py rename to qualification_tests/benchmark_qualification/catalog.py index 020daba04b..548639dafa 100644 --- a/tools/benchmark_qualification/catalog.py +++ b/qualification_tests/benchmark_qualification/catalog.py @@ -5,8 +5,9 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field as dataclass_field from pathlib import Path +import re from typing import Any, Mapping, Sequence import yaml @@ -18,6 +19,35 @@ class QualificationError(RuntimeError): pass +class _UniqueKeyLoader(yaml.SafeLoader): + """Load policy YAML while rejecting duplicate explicit mapping keys.""" + + def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Any, Any]: + seen: set[Any] = set() + for key_node, _ in node.value: + if key_node.tag == "tag:yaml.org,2002:merge": + continue + key = self.construct_object(key_node, deep=False) + try: + duplicate = key in seen + seen.add(key) + except TypeError as error: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable mapping key", + key_node.start_mark, + ) from error + if duplicate: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + return super().construct_mapping(node, deep=deep) + + @dataclass(frozen=True) class QualificationCase: kind: str @@ -29,6 +59,9 @@ class QualificationCase: values: Mapping[str, Any] source: Path reference_requirements: Path | None + reference_paths: Mapping[str, str] = dataclass_field(default_factory=dict) + reference_build_isolation: bool = True + environment_hook: Path | None = None @property def id(self) -> str: @@ -50,7 +83,12 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: if path.stem != model: raise QualificationError(f"{path}: file name must match model {model!r}") candidate = _candidate(raw.get("candidate"), family, path) - requirements = _requirements(raw.get("reference_environment"), path) + reference_environment = raw.get("reference_environment") + requirements = _requirements(reference_environment, path) + reference_paths = _reference_paths(reference_environment, path) + build_isolation = _build_isolation(reference_environment, path) + hook_path = path.parent / "prepare_environment.py" + environment_hook = hook_path.resolve() if hook_path.is_file() else None for kind in ("accuracy", "performance"): configured = raw.get(kind, []) if not isinstance(configured, list): @@ -70,6 +108,9 @@ def discover(repository: Path) -> tuple[QualificationCase, ...]: values=dict(value), source=path.resolve(), reference_requirements=requirements, + reference_paths=reference_paths, + reference_build_isolation=build_isolation, + environment_hook=environment_hook, ) if case.id in seen: raise QualificationError(f"duplicate qualification case {case.id!r}") @@ -103,7 +144,7 @@ def select( def load_benchmark(repository: Path, case: QualificationCase) -> dict[str, Any]: - path = repository / "tools/benchmark_qualification/benchmarks" / f"{case.benchmark}.yaml" + path = repository / "qualification_tests/benchmark_qualification/benchmarks" / f"{case.benchmark}.yaml" value = _yaml_object(path, "benchmark definition") if value.get("kind") != case.kind: raise QualificationError( @@ -123,9 +164,34 @@ def _candidate(raw: Any, family: str, path: Path) -> dict[str, Any]: ) for field in ("checkpoint", "task", "precision"): _string(value.get(field), f"candidate.{field}", path) + selected_task = value.get("selected_task") + if selected_task is not None: + _string(selected_task, "candidate.selected_task", path) + model_directory = value.get("model_directory") + if model_directory is not None: + configured = Path(_string(model_directory, "candidate.model_directory", path)) + if configured.is_absolute() or ".." in configured.parts: + raise QualificationError( + f"{path}: candidate.model_directory must stay inside the family environment" + ) revision = value.get("revision") if revision is not None and not isinstance(revision, str): raise QualificationError(f"{path}: candidate.revision must be a string") + trust_remote_code = value.get("trust_remote_code") + if trust_remote_code is not None and not isinstance(trust_remote_code, bool): + raise QualificationError(f"{path}: candidate.trust_remote_code must be a boolean") + if trust_remote_code and ( + not isinstance(revision, str) or re.fullmatch(r"[0-9a-fA-F]{40}", revision) is None + ): + raise QualificationError( + f"{path}: trusted remote code requires an immutable 40-character revision" + ) + if model_directory is None and ( + not isinstance(revision, str) or re.fullmatch(r"[0-9a-fA-F]{40}", revision) is None + ): + raise QualificationError( + f"{path}: remote checkpoint requires an immutable 40-character revision" + ) build = value.get("build", {}) if not isinstance(build, Mapping): raise QualificationError(f"{path}: candidate.build must be an object") @@ -149,9 +215,41 @@ def _requirements(raw: Any, path: Path) -> Path | None: return result +def _build_isolation(raw: Any, path: Path) -> bool: + if raw is None: + return True + if not isinstance(raw, Mapping): + raise QualificationError(f"{path}: reference_environment must be an object") + value = raw.get("build_isolation", True) + if not isinstance(value, bool): + raise QualificationError(f"{path}: reference_environment.build_isolation must be a boolean") + return value + + +def _reference_paths(raw: Any, path: Path) -> dict[str, str]: + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise QualificationError(f"{path}: reference_environment must be an object") + configured = raw.get("paths", {}) + if not isinstance(configured, Mapping): + raise QualificationError(f"{path}: reference_environment.paths must be an object") + result = {} + for name, value in configured.items(): + if not isinstance(name, str) or not name: + raise QualificationError(f"{path}: reference_environment.paths keys must be strings") + relative = Path(_string(value, f"reference_environment.paths.{name}", path)) + if relative.is_absolute() or ".." in relative.parts: + raise QualificationError( + f"{path}: reference_environment.paths.{name} must stay inside the environment" + ) + result[name] = relative.as_posix() + return result + + def _yaml_object(path: Path, label: str) -> dict[str, Any]: try: - value = yaml.safe_load(path.read_text(encoding="utf-8")) + value = yaml.load(path.read_text(encoding="utf-8"), Loader=_UniqueKeyLoader) except (OSError, yaml.YAMLError) as error: raise QualificationError(f"cannot read {label} {path}: {error}") from error if not isinstance(value, dict): diff --git a/tools/benchmark_qualification/datasets.py b/qualification_tests/benchmark_qualification/datasets.py similarity index 84% rename from tools/benchmark_qualification/datasets.py rename to qualification_tests/benchmark_qualification/datasets.py index 945579e405..37b9edd6bb 100644 --- a/tools/benchmark_qualification/datasets.py +++ b/qualification_tests/benchmark_qualification/datasets.py @@ -33,7 +33,11 @@ def receipt(self) -> dict[str, Any]: } -def resolve_dataset(definition: Mapping[str, Any], context: RuntimeContext) -> Dataset: +def resolve_dataset( + definition: Mapping[str, Any], + context: RuntimeContext, + profile: Path | None = None, +) -> Dataset: configured = definition.get("dataset") if not isinstance(configured, Mapping): raise QualificationError("Accuracy benchmark requires a dataset object") @@ -62,9 +66,21 @@ def resolve_dataset(definition: Mapping[str, Any], context: RuntimeContext) -> D raise QualificationError(f"downloadable dataset {dataset_id!r} requires dataset.sha256") path = _download_dataset(dataset_id, source, context.data_root) resolved_mode = "download" + elif mode == "family": + relative = source.get("path") + if profile is None: + raise QualificationError(f"family dataset {dataset_id!r} requires its model profile") + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise QualificationError("family dataset source.path must be relative") + family_root = profile.parents[2].resolve() + path = (profile.parent / relative).resolve() + if not path.is_relative_to(family_root): + raise QualificationError("family dataset source.path must stay inside its family") + resolved_mode = "family" else: raise QualificationError( - f"dataset {dataset_id!r} has unsupported source mode {mode!r}; use manual or download" + f"dataset {dataset_id!r} has unsupported source mode {mode!r}; " + "use manual, download, or family" ) if not path.is_file(): if mode == "manual": diff --git a/qualification_tests/benchmark_qualification/performance/README.md b/qualification_tests/benchmark_qualification/performance/README.md new file mode 100644 index 0000000000..085c6fa31b --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/README.md @@ -0,0 +1,95 @@ + + +# Performance qualification + +This package is repository-only CI qualification. The installed user benchmark +remains `trtmc-bench` under `apps/benchmark`; it does not import this package. + +## Ownership + +- `matrix.py` selects and resumes multi-model campaigns. +- `runner.py` executes one resolved qualification case. +- `reference_protocol.py` defines the Accuracy/Performance reference command. +- `reference_harness.py` provides model-agnostic timing and result writing for + family-owned references. +- `reporting.py` produces `report.json` and `report.html`. +- `families//tests/benchmark/*.yaml` owns model cases, datasets, + metrics, gates, reference environment, and measurement settings. +- `families//tests/benchmark/reference.py` owns reference behavior that + cannot be expressed by a framework-generic adapter. + +Shared code must not select behavior from a family name. Adding a normal family +case changes only the family directory. Campaign configuration may select cases +or hardware, but does not redefine model behavior. + +## Execution + +The candidate runs through the installed `trtmc-bench` executable. The +reference runs in the Python environment prepared for its family, so conflicting +reference dependencies do not affect the candidate or another family. + +Run a checked-in campaign entry with: + +```bash +python3 -m qualification_tests.benchmark_qualification.performance check \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ + --entry gpt2.generate + +python3 -m qualification_tests.benchmark_qualification.performance run \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ + --entry gpt2.generate +``` + +Resume a run or regenerate its report with: + +```bash +python3 -m qualification_tests.benchmark_qualification.performance resume +python3 -m qualification_tests.benchmark_qualification.performance report +``` + +`tools/model_benchmark.py` is the thin per-model CI entry point. It discovers +family profiles and invokes the same single-case runner; it does not build a +temporary campaign or start a nested matrix process. + +## Reference contract + +A reference declares exactly one of: + +```yaml +reference: + adapter: hf-transformers-asr +``` + +or: + +```yaml +reference: + script: tests/benchmark/reference.py +``` + +Framework-generic adapters receive only the declared adapter and parameters. +Family scripts receive their own family identity but shared code never branches +on it. Both return `trtmc.perf-baseline/v1` with one finite positive timing +sample per measured iteration and the output summary required by the declared +contract. + +Reference model loading occurs before warmup. The three timing fields +`timing_scope`, `input_preparation_included`, and `asset_loading_included` must +match the profile. Model load, compilation, and warmup are excluded from timed +samples. + +## Results + +Each run retains raw commands, logs, artifacts, and `results.json`, then writes: + +- `report.json` for automation; +- `report.html` for inspection. + +An output-contract mismatch is a model `failed` result. Reference/candidate +command failures, invalid reports, missing artifacts, and other execution +problems are `error`, not model gate failures. diff --git a/qualification_tests/benchmark_qualification/performance/__init__.py b/qualification_tests/benchmark_qualification/performance/__init__.py new file mode 100644 index 0000000000..cbe4fa2c76 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Internal Performance qualification and campaign execution.""" + +from .matrix import main +from .runner import run_case + +__all__ = ["main", "run_case"] diff --git a/qualification_tests/benchmark_qualification/performance/__main__.py b/qualification_tests/benchmark_qualification/performance/__main__.py new file mode 100644 index 0000000000..b2aee43827 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/__main__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from .matrix import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/benchmark/performance/environments/auto-thor.yaml b/qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml similarity index 82% rename from apps/benchmark/performance/environments/auto-thor.yaml rename to qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml index 0b0d25f8c4..f14916b0a9 100644 --- a/apps/benchmark/performance/environments/auto-thor.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml @@ -7,8 +7,8 @@ name: auto-thor tools: trtmc_bench: apps/benchmark/trtmc-bench trtmc_worker: ${TRTMC_PERF_WORKER} - hf_transformers_runner: apps/benchmark/performance/baselines/hf_transformers.py - task_reference_runner: apps/benchmark/performance/baselines/task_reference.py + hf_transformers_runner: qualification_tests/benchmark_qualification/performance/references/hf_transformers.py + task_reference_runner: qualification_tests/benchmark_qualification/performance/references/generic_reference.py references: elf_repo: ${TRTMC_ELF_REFERENCE_REPO} diff --git a/apps/benchmark/performance/environments/gb300.yaml b/qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml similarity index 81% rename from apps/benchmark/performance/environments/gb300.yaml rename to qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml index 6eb03c0a66..6e991421b8 100644 --- a/apps/benchmark/performance/environments/gb300.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml @@ -7,8 +7,8 @@ name: gb300-ci tools: trtmc_bench: apps/benchmark/trtmc-bench trtmc_worker: ${TRTMC_PERF_WORKER} - hf_transformers_runner: apps/benchmark/performance/baselines/hf_transformers.py - task_reference_runner: apps/benchmark/performance/baselines/task_reference.py + hf_transformers_runner: qualification_tests/benchmark_qualification/performance/references/hf_transformers.py + task_reference_runner: qualification_tests/benchmark_qualification/performance/references/generic_reference.py references: elf_repo: ${TRTMC_ELF_REFERENCE_REPO} diff --git a/apps/benchmark/performance/environments/l4t-thor.yaml b/qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml similarity index 82% rename from apps/benchmark/performance/environments/l4t-thor.yaml rename to qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml index 8744f6191b..f536ff145e 100644 --- a/apps/benchmark/performance/environments/l4t-thor.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml @@ -7,8 +7,8 @@ name: l4t-thor tools: trtmc_bench: apps/benchmark/trtmc-bench trtmc_worker: ${TRTMC_PERF_WORKER} - hf_transformers_runner: apps/benchmark/performance/baselines/hf_transformers.py - task_reference_runner: apps/benchmark/performance/baselines/task_reference.py + hf_transformers_runner: qualification_tests/benchmark_qualification/performance/references/hf_transformers.py + task_reference_runner: qualification_tests/benchmark_qualification/performance/references/generic_reference.py references: elf_repo: ${TRTMC_ELF_REFERENCE_REPO} diff --git a/apps/benchmark/performance/release.yaml b/qualification_tests/benchmark_qualification/performance/config/release.yaml similarity index 91% rename from apps/benchmark/performance/release.yaml rename to qualification_tests/benchmark_qualification/performance/config/release.yaml index 28ea3b5f9a..f891fda4be 100644 --- a/apps/benchmark/performance/release.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/release.yaml @@ -18,6 +18,10 @@ defaults: asset_loading_included: false excluded_profiles: + - model: moge-2-vitl + reason: >- + Family-owned accuracy and internal performance qualification are present, + but no user-facing release-performance workload or receipt has been added. - model: s1-mini-fp16 reason: >- families/s1_mini has no release-performance workload or receipt yet; @@ -143,7 +147,7 @@ entries: testcase: detr-resnet-50 baseline: runner: task-reference - adapter: hf-transformers-vision + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -228,7 +232,7 @@ entries: testcase: canary-1b-v2 baseline: runner: task-reference - adapter: nemo-asr + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -242,6 +246,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: chronos-bolt} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -285,7 +290,7 @@ entries: testcase: deepseek-ocr baseline: runner: task-reference - adapter: hf-transformers-vlm + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-pipeline-call-wall @@ -318,7 +323,7 @@ entries: testcase: dinov3-vits16-pretrain-lvd1689m baseline: runner: task-reference - adapter: hf-transformers-vision + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -407,7 +412,11 @@ entries: valid_iters: 8 baseline: runner: task-reference - adapter: upstream-fast-foundation-stereo + script: tests/benchmark/reference.py + reference_inputs: + model_dir: + environment: fast_foundation_stereo_model + required: [core/foundation_stereo.py, core/submodule.py, weights/23-36-37/model_best_bp2_serialize.pth] mode: pytorch-eager reference_backend: official_pytorch timing_scope: task-pipeline-call-wall @@ -533,6 +542,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {model_class: AutoModel} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -546,7 +556,9 @@ entries: testcase: lance-3b-x2t-image baseline: runner: task-reference - adapter: upstream-lance + script: tests/benchmark/reference.py + reference_inputs: + reference_repo: {environment: lance_repo, required: [inference_lance.py]} mode: pytorch-eager reference_backend: upstream_lance timing_scope: task-pipeline-call-wall @@ -564,7 +576,9 @@ entries: testcase: act-aloha-sim-transfer-cube-recorded-episode-0-frame-0 baseline: runner: task-reference - adapter: pytorch-lerobot-act + script: tests/benchmark/reference.py + reference_inputs: + source_root: {environment: lerobot_repo, required: [lerobot/common/policies/act/modeling_act.py]} mode: pytorch-eager precision: fp32 reference_backend: official_lerobot @@ -590,7 +604,7 @@ entries: testcase: locateanything-3b baseline: runner: task-reference - adapter: hf-transformers-vlm + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -621,7 +635,7 @@ entries: max_new_tokens: 256 baseline: runner: task-reference - adapter: nemo-tts + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -729,7 +743,7 @@ entries: testcase: nemotron-3.5-asr-streaming-0.6b baseline: runner: task-reference - adapter: nemo-asr + script: tests/benchmark/reference.py mode: pytorch-eager reference_backend: nemo_reference timing_scope: task-pipeline-call-wall @@ -775,6 +789,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtsmixer} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -788,6 +803,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: patchtst} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -800,7 +816,9 @@ entries: testcase: personaplex-7b baseline: runner: task-reference - adapter: pytorch-personaplex + script: tests/benchmark/reference.py + reference_inputs: + official_repo: {environment: personaplex_repo, required: [moshi]} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-pipeline-call-wall @@ -825,6 +843,10 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: + model_class: AutoModelForCausalLM + workflow: single-image-token-chat + eager_attention: true mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -897,7 +919,7 @@ entries: testcase: qwen3-omni-30b-a3b-instruct baseline: runner: task-reference - adapter: hf-qwen3-omni + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -918,6 +940,8 @@ entries: output_contract: media-shape adapter_options: cpu_offload: true + always_negative_prompt: true + guidance_parameter: true_cfg_scale - id: qwen_moe.generate family: qwen_moe operation: generate @@ -938,6 +962,7 @@ entries: baseline: runner: task-reference adapter: hf-transformers-vlm + adapter_options: {model_class: AutoModelForImageTextToText} mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -970,7 +995,7 @@ entries: testcase: sam-vit-base baseline: runner: task-reference - adapter: hf-transformers-vision + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -984,7 +1009,7 @@ entries: testcase: sam3 baseline: runner: task-reference - adapter: hf-transformers-vision + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -999,13 +1024,21 @@ entries: testcase: sana-wm-bidirectional baseline: runner: task-reference - adapter: upstream-sana-wm + script: tests/benchmark/reference.py + reference_inputs: + reference_repo: {environment: sana_repo} + model_dir: {environment: sana_model} mode: pytorch-eager reference_backend: official_sana timing_scope: task-pipeline-call-wall output_contract: media-shape adapter_options: intrinsics: assets/demo_0_intrinsics.npy + translation_speed: 0.055 + rotation_speed_deg: 1.2 + fps: 16 + flow_shift: 9.8 + no_action_overlay: true - id: segformer.segment family: segformer operation: segment @@ -1014,7 +1047,7 @@ entries: testcase: segformer-b0-ade baseline: runner: task-reference - adapter: hf-transformers-vision + script: tests/benchmark/reference.py mode: hf-eager reference_backend: hf_transformers timing_scope: task-model-call-wall @@ -1060,6 +1093,7 @@ entries: baseline: runner: task-reference adapter: pytorch-timeseries + adapter_options: {reference_type: timesfm} mode: pytorch-eager reference_backend: torch_reference timing_scope: task-model-call-wall @@ -1072,9 +1106,9 @@ entries: testcase: timm-vit-base-p16-224-augreg-in21k-ft-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_resnet.classify @@ -1085,9 +1119,9 @@ entries: testcase: resnet50-a1-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_xception.classify @@ -1098,9 +1132,9 @@ entries: testcase: xception41-tf-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_regnet.classify @@ -1111,9 +1145,9 @@ entries: testcase: regnety-040-ra3-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_seresnet.classify @@ -1124,9 +1158,9 @@ entries: testcase: seresnet50-a1-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_senet.classify @@ -1137,9 +1171,9 @@ entries: testcase: senet154-gluon-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_ghostnet.classify @@ -1150,9 +1184,9 @@ entries: testcase: ghostnet-100-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_convnext.classify @@ -1163,9 +1197,9 @@ entries: testcase: convnext-tiny-in12k-ft-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_inception_v4.classify @@ -1176,9 +1210,9 @@ entries: testcase: inception-v4-tf-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_inception_resnet.classify @@ -1189,9 +1223,9 @@ entries: testcase: inception-resnet-v2-tf-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_hrnet.classify @@ -1202,9 +1236,9 @@ entries: testcase: hrnet-w18-ms-aug-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_resnest.classify @@ -1215,9 +1249,9 @@ entries: testcase: resnest50d-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_swin.classify @@ -1228,9 +1262,9 @@ entries: testcase: swin-tiny-patch4-window7-224-ms-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_crossvit.classify @@ -1241,9 +1275,9 @@ entries: testcase: crossvit-9-240-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_dpn.classify @@ -1254,9 +1288,9 @@ entries: testcase: dpn68b-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_res2net.classify @@ -1267,9 +1301,9 @@ entries: testcase: res2net50-26w-4s-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: yolov10.detect @@ -1293,9 +1327,9 @@ entries: testcase: dm-nfnet-f0-dm-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilevit.classify @@ -1306,9 +1340,9 @@ entries: testcase: mobilevit-s-cvnets-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: yolov8.detect @@ -1345,9 +1379,9 @@ entries: testcase: xcit-tiny-12-p16-224-fb-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_repvgg.classify @@ -1358,9 +1392,9 @@ entries: testcase: repvgg-a2-rvgg-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_inception.classify @@ -1371,9 +1405,9 @@ entries: testcase: inception-v3-tv-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mnasnet.classify @@ -1384,9 +1418,9 @@ entries: testcase: mnasnet-100-rmsp-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilenetv2.classify @@ -1397,9 +1431,9 @@ entries: testcase: mobilenetv2-100-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_densenet.classify @@ -1410,9 +1444,9 @@ entries: testcase: densenet121-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_efficientnet.classify @@ -1423,9 +1457,9 @@ entries: testcase: efficientnet-b0-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilenetv4.classify @@ -1436,9 +1470,9 @@ entries: testcase: mobilenetv4-conv-small-050 baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_mobilenetv3.classify @@ -1449,9 +1483,9 @@ entries: testcase: mobilenetv3-large-100-ra-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: timm_vgg.classify @@ -1462,9 +1496,9 @@ entries: testcase: vgg16-tv-in1k baseline: runner: task-reference - adapter: hf-transformers-vision + adapter: timm-classification mode: hf-eager - reference_backend: hf_transformers + reference_backend: timm timing_scope: task-model-call-wall input_preparation_included: false - id: wan_t2v.generate_image @@ -1498,6 +1532,8 @@ entries: adapter_options: model_id: Wan-AI/Wan2.2-TI2V-5B-Diffusers model_revision: b8fff7315c768468a5333511427288870b2e9635 + vae_class: AutoencoderKLWan + vae_precision: fp32 - id: whisper.transcribe family: whisper operation: transcribe @@ -1582,6 +1618,8 @@ additional_profiles: inherit: llama.generate - model: nemotron-speech-streaming-en-0.6b inherit: nemotron_speech_streaming.transcribe + baseline: + adapter_options: {transcription_input: manifest, model_loader: nemo-pretrained} - model: paraphrase-multilingual-minilm-l12-v2 inherit: bert.encode - model: patchtst-granite-official diff --git a/apps/benchmark/performance/requirements.txt b/qualification_tests/benchmark_qualification/performance/config/requirements.txt similarity index 100% rename from apps/benchmark/performance/requirements.txt rename to qualification_tests/benchmark_qualification/performance/config/requirements.txt diff --git a/tools/perf_matrix.py b/qualification_tests/benchmark_qualification/performance/matrix.py similarity index 84% rename from tools/perf_matrix.py rename to qualification_tests/benchmark_qualification/performance/matrix.py index 460f85aa2f..7e25d1ed46 100755 --- a/tools/perf_matrix.py +++ b/qualification_tests/benchmark_qualification/performance/matrix.py @@ -7,7 +7,6 @@ from __future__ import annotations import argparse -import html import json import math import os @@ -21,7 +20,6 @@ import time from array import array from copy import deepcopy -from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from string import Template @@ -30,7 +28,7 @@ import yaml -REPOSITORY = Path(__file__).resolve().parents[1] +REPOSITORY = Path(__file__).resolve().parents[3] BUILDER_SOURCE = REPOSITORY / "core/builder" BENCHMARK_SOURCE = REPOSITORY / "apps/benchmark" MANIFEST_ROOT = REPOSITORY / "families" @@ -38,11 +36,29 @@ if str(source) not in sys.path: sys.path.insert(0, str(source)) -from apps.benchmark.performance.baselines.timing_contracts import timing_contract # noqa: E402 -from apps.benchmark.performance.baselines.hf_transformers import flatten_config # noqa: E402 +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract # noqa: E402 +from qualification_tests.benchmark_qualification.performance.references.hf_transformers import flatten_config # noqa: E402 +from qualification_tests.benchmark_qualification.performance import reference_protocol # noqa: E402 +from qualification_tests.benchmark_qualification.performance.reporting import write_report # noqa: E402 +from trtmc_benchmark.artifact_metrics import ( # noqa: E402 + metric_geometry_metrics, + metric_geometry_passes, + robot_action_metrics, + robot_action_passes, +) from trtmc_benchmark.catalog import ManifestCatalog, resolve_case, selected_task_for_case # noqa: E402 from trtmc_benchmark.task_adapters import default_operation # noqa: E402 from trtmc_benchmark.types import BenchmarkError # noqa: E402 +from qualification_tests.benchmark_qualification.performance.stability import ( # noqa: E402 + measurement_stability as _measurement_stability, + p50 as _p50, + timing_stability as _timing_stability, # noqa: F401 - compatibility for existing tests +) +from qualification_tests.benchmark_qualification.performance.types import ( # noqa: E402 + Environment, + PerfMatrixError, + ResolvedEntry, +) SUITE_SCHEMA = "trtmc.perf-suite/v2" @@ -65,6 +81,7 @@ OUTPUT_CONTRACTS = { "audio-shape", "classification-top-class", + "detection-parity", "disparity-parity", "embedding-shape", "exact-text", @@ -76,77 +93,24 @@ "regression-values", "generated-token-count", "image-features-shape", + "instance-mask-parity", "localization", "media-shape", "metric-geometry-shape", + "metric-geometry-parity", "molecular-structure-shape", "normalized-text", "ocr-text", "offline-speech-shape", "pose-refinement-shape", + "prompted-mask-parity", "reranking-order", "robot-action-shape", + "robot-action-parity", "segmentation-shape", "transcription-text", + "vision-language-text", } -REFERENCE_INPUTS = { - "pytorch-lerobot-act": (("source_root", "lerobot_repo"),), - "upstream-elf": (("reference_repo", "elf_repo"),), - "upstream-lance": (("reference_repo", "lance_repo"),), - "upstream-sana-wm": ( - ("reference_repo", "sana_repo"), - ("model_dir", "sana_model"), - ), - "pytorch-personaplex": (("official_repo", "personaplex_repo"),), - "upstream-fast-foundation-stereo": (("model_dir", "fast_foundation_stereo_model"),), -} -REFERENCE_FIELDS = { - "elf_repo", - "lance_repo", - "lerobot_repo", - "sana_repo", - "sana_model", - "personaplex_repo", - "fast_foundation_stereo_model", -} - - -class PerfMatrixError(RuntimeError): - pass - - -@dataclass(frozen=True) -class Environment: - name: str - trtmc_bench: Path - worker: Path - hf_runner: Path - task_runner: Path - results_root: Path - scratch_root: Path - bundle_cache: Path - bundle_roots: tuple[Path, ...] - runtime_root: Path - bundle_retention: str - local_files_only: bool - timeout_seconds: int - references: Mapping[str, str] - reference_python: Path = Path(sys.executable) - storage_root: Path | None = None - hf_cache_mode: str = "shared" - hf_cache_retention: str = "retain" - - -@dataclass(frozen=True) -class ResolvedEntry: - spec: Mapping[str, Any] - model: Any - case: Any - manifest: Mapping[str, Any] - reference_precision: str - baseline_timing: Mapping[str, Any] - - def parser() -> argparse.ArgumentParser: value = argparse.ArgumentParser(description=__doc__) commands = value.add_subparsers(dest="command", required=True) @@ -207,7 +171,7 @@ def load_suite(path: Path) -> tuple[str, list[dict[str, Any]], set[str]]: if owner is not None: _family_file(owner, "tests/performance.yaml", "performance suite") name, entries, excluded = _load_suite_file(path, owner=owner) - canonical = REPOSITORY / "apps/benchmark/performance/release.yaml" + canonical = REPOSITORY / "qualification_tests/benchmark_qualification/performance/config/release.yaml" if path.resolve() != canonical.resolve(): return name, entries, excluded positions = {entry["id"]: index for index, entry in enumerate(entries)} @@ -371,9 +335,47 @@ def _validate_entry(entry: Mapping[str, Any]) -> None: _family_script(entry) elif baseline["runner"] == "task-reference" and not isinstance(baseline.get("adapter"), str): raise PerfMatrixError(f"entry {entry['id']} requires baseline.adapter") + reference_inputs = baseline.get("reference_inputs", {}) + if not isinstance(reference_inputs, Mapping): + raise PerfMatrixError(f"entry {entry['id']} baseline.reference_inputs must be an object") + for option_name, declaration in reference_inputs.items(): + if not isinstance(option_name, str) or not option_name: + raise PerfMatrixError( + f"entry {entry['id']} baseline.reference_inputs keys must be non-empty strings" + ) + if not isinstance(declaration, Mapping): + raise PerfMatrixError( + f"entry {entry['id']} reference input {option_name!r} must be an object" + ) + environment_field = declaration.get("environment") + required = declaration.get("required", []) + if not isinstance(environment_field, str) or not environment_field: + raise PerfMatrixError( + f"entry {entry['id']} reference input {option_name!r} requires environment" + ) + if not isinstance(required, list) or not all( + isinstance(path, str) + and path + and not Path(path).is_absolute() + and ".." not in Path(path).parts + for path in required + ): + raise PerfMatrixError( + f"entry {entry['id']} reference input {option_name!r} required files " + "must be relative paths" + ) fallback = baseline.get("fallback") if fallback is not None and (not isinstance(fallback, str) or not fallback): raise PerfMatrixError(f"entry {entry['id']} baseline.fallback must be a mode") + source_placement = baseline.get("source_language_placement") + if source_placement is not None and ( + baseline["runner"] != "hf-transformers" + or source_placement != "replace-final-unk" + ): + raise PerfMatrixError( + f"entry {entry['id']} baseline.source_language_placement must be " + "replace-final-unk on an hf-transformers reference" + ) if not isinstance(measurement, Mapping): raise PerfMatrixError(f"entry {entry['id']} requires measurement") warmup = measurement.get("warmup") @@ -453,10 +455,11 @@ def load_environment(path: Path) -> Environment: raise PerfMatrixError("environment tools is missing: " + ", ".join(missing)) if missing := sorted(required_storage - storage.keys()): raise PerfMatrixError("environment storage is missing: " + ", ".join(missing)) - if missing := sorted(REFERENCE_FIELDS - references.keys()): - raise PerfMatrixError("environment references is missing: " + ", ".join(missing)) - if not all(isinstance(references[name], str) for name in REFERENCE_FIELDS): - raise PerfMatrixError("environment reference inputs must be strings") + if not all( + isinstance(name, str) and name and isinstance(value, str) + for name, value in references.items() + ): + raise PerfMatrixError("environment reference inputs must map names to strings") timeout = execution.get("timeout_seconds", 7200) if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout < 1: raise PerfMatrixError("execution.timeout_seconds must be positive") @@ -494,7 +497,7 @@ def load_environment(path: Path) -> Environment: bundle_retention=retention, local_files_only=bool(execution.get("local_files_only", False)), timeout_seconds=timeout, - references={name: str(references[name]) for name in REFERENCE_FIELDS}, + references={str(name): str(value) for name, value in references.items()}, storage_root=( _path(storage_root, "storage.storage_root") if storage_root is not None else None ), @@ -791,41 +794,53 @@ def _adapter_options(entry: ResolvedEntry, environment: Environment) -> dict[str if not isinstance(configured, Mapping): raise PerfMatrixError(f"entry {entry.spec['id']} adapter_options must be an object") options = dict(configured) - inputs = REFERENCE_INPUTS.get(str(entry.spec["baseline"].get("adapter", "")), ()) - for option_name, field in inputs: + inputs = entry.spec["baseline"].get("reference_inputs", {}) + for option_name, declaration in inputs.items(): + field = str(declaration["environment"]) + if field not in environment.references: + raise PerfMatrixError( + f"entry {entry.spec['id']} environment references is missing: {field}" + ) path = _path(environment.references[field], f"references.{field}") if not path.is_dir(): raise PerfMatrixError( f"entry {entry.spec['id']} reference path is not a directory: {path}" ) - _validate_reference_path(entry, field, path) + missing = [ + relative + for relative in declaration.get("required", []) + if not (path / relative).exists() + ] + if missing: + raise PerfMatrixError( + f"entry {entry.spec['id']} reference path {path} is missing: " + ", ".join(missing) + ) options[option_name] = str(path) return options -def _validate_reference_path(entry: ResolvedEntry, field: str, path: Path) -> None: - required = { - "elf_repo": ("src",), - "lance_repo": ("inference_lance.py",), - "lerobot_repo": ("lerobot/common/policies/act/modeling_act.py",), - "personaplex_repo": ("moshi",), - "fast_foundation_stereo_model": ( - "core/foundation_stereo.py", - "core/submodule.py", - "weights/23-36-37/model_best_bp2_serialize.pth", - ), - }.get(field, ()) - missing = [relative for relative in required if not (path / relative).exists()] - if missing: - raise PerfMatrixError( - f"entry {entry.spec['id']} reference path {path} is missing: " + ", ".join(missing) - ) - - def _baseline_mode(baseline: Mapping[str, Any]) -> str: return str(baseline.get("mode", "reference" if "script" in baseline else "torch-compile")) +def _baseline_model(entry: ResolvedEntry, environment: Environment) -> str: + configured = entry.spec["baseline"].get("model") + if configured is None: + configured = _adapter_options(entry, environment).get("model_id", entry.model.hf_id) + if not isinstance(configured, str) or not configured: + raise PerfMatrixError(f"entry {entry.spec['id']} baseline model must be non-empty") + return configured + + +def _baseline_revision(entry: ResolvedEntry, model: str) -> str | None: + configured = entry.spec["baseline"].get("revision") + if configured is None and model == entry.model.hf_id: + configured = entry.model.hf_revision or None + if configured is not None and (not isinstance(configured, str) or not configured): + raise PerfMatrixError(f"entry {entry.spec['id']} baseline revision must be non-empty") + return configured + + def baseline_command(entry: ResolvedEntry, environment: Environment, output: Path) -> list[str]: return _baseline_command(entry, environment, output) @@ -840,12 +855,19 @@ def _baseline_command( baseline = entry.spec["baseline"] runner = str(baseline["runner"]) selected_mode = str(mode or _baseline_mode(baseline)) - request = json.dumps(flatten_config(entry.case.request), ensure_ascii=True, separators=(",", ":")) + request_value = flatten_config(entry.case.request) + source_placement = baseline.get("source_language_placement") + if source_placement is not None: + request_value["source_language_placement"] = source_placement + model = _baseline_model(entry, environment) + revision = _baseline_revision(entry, model) + local_files_only = bool(baseline.get("local_files_only", environment.local_files_only)) + trust_remote_code = bool(entry.manifest.get("trust_remote_code", False)) common = [ "--model", - str(_adapter_options(entry, environment).get("model_id", entry.model.hf_id)), + model, "--request-json", - request, + json.dumps(request_value, ensure_ascii=True, separators=(",", ":")), "--precision", entry.reference_precision, "--mode", @@ -886,42 +908,40 @@ def _baseline_command( if bool(baseline.get("dynamic", True)): arguments.append("--compile-dynamic") else: - executable = ( - [str(_family_script(entry.spec))] - if "script" in baseline - else [str(environment.task_runner), "--adapter", str(baseline["adapter"])] - ) - arguments = [ - str(getattr(environment, "reference_python", sys.executable)), - *executable, - "--family", - entry.model.family, - "--operation", - str(entry.spec["operation"]), - "--manifest", - str(entry.model.manifest_path), - "--adapter-options-json", - json.dumps( - _adapter_options(entry, environment), - ensure_ascii=True, - separators=(",", ":"), + return reference_protocol.command( + python=getattr(environment, "reference_python", Path(sys.executable)), + generic_runner=environment.task_runner, + script=_family_script(entry.spec) if "script" in baseline else None, + family=entry.model.family, + operation=str(entry.spec["operation"]), + manifest=entry.model.manifest_path, + selected_task=( + _effective_task(entry) + if "script" in baseline or entry.case.selected_task is not None + else None ), - "--timing-contract-json", - json.dumps(entry.baseline_timing, ensure_ascii=True, separators=(",", ":")), - "--padding", - str(baseline.get("padding", "longest")), - *common, - ] - if "script" not in baseline: - arguments.extend(("--testcase-name", entry.case.testcase_name)) - if "script" in baseline or entry.case.selected_task is not None: - arguments.extend(("--selected-task", _effective_task(entry))) - revision = entry.model.hf_revision + testcase_name=entry.case.testcase_name, + adapter=None if "script" in baseline else str(baseline["adapter"]), + adapter_options=_adapter_options(entry, environment), + timing_contract=entry.baseline_timing, + padding=str(baseline.get("padding", "longest")), + model=model, + revision=revision, + request=request_value, + precision=entry.reference_precision, + mode=selected_mode, + warmup=entry.case.measurement.warmup, + iterations=entry.case.measurement.iterations, + case_name=str(entry.spec["id"]), + output=output, + trust_remote_code=trust_remote_code, + local_files_only=local_files_only, + ) if revision: arguments.extend(("--revision", revision)) - if bool(entry.manifest.get("trust_remote_code", False)): + if trust_remote_code: arguments.append("--trust-remote-code") - if bool(baseline.get("local_files_only", environment.local_files_only)): + if local_files_only: arguments.append("--local-files-only") return arguments @@ -932,7 +952,7 @@ def _validate_script_result( expected = { "schema_version": "trtmc.perf-baseline/v1", "status": "completed", - "model": str(_adapter_options(entry, environment).get("model_id", entry.model.hf_id)), + "model": _baseline_model(entry, environment), "family": entry.model.family, "operation": str(entry.spec["operation"]), "case_name": str(entry.spec["id"]), @@ -1077,94 +1097,6 @@ def _candidate_result(path: Path) -> dict[str, Any]: } -def _p50(value: Mapping[str, Any]) -> float: - metrics = value.get("metrics", {}) - latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} - p50 = latency.get("p50") if isinstance(latency, Mapping) else None - if isinstance(p50, bool) or not isinstance(p50, (int, float)) or not math.isfinite(float(p50)): - raise PerfMatrixError("measurement has no finite latency p50") - return float(p50) - - -_TIMING_STABILITY_SAMPLE_COUNT = 10 -_TIMING_STABILITY_MAX_HALF_CHANGE_PERCENT = 5.0 -_TIMING_STABILITY_MEDIAN_BAND_PERCENT = 5.0 -_TIMING_STABILITY_MIN_IN_BAND = 8 - - -def _timing_stability(values: Sequence[Any]) -> dict[str, Any]: - if len(values) != _TIMING_STABILITY_SAMPLE_COUNT: - return { - "status": "not_evaluated", - "sample_count": len(values), - "reason": "requires_10_samples", - } - try: - samples = [float(value) for value in values] - except (TypeError, ValueError): - return { - "status": "not_evaluated", - "sample_count": len(values), - "reason": "invalid_samples", - } - if not all(math.isfinite(value) and value > 0.0 for value in samples): - return { - "status": "not_evaluated", - "sample_count": len(samples), - "reason": "invalid_samples", - } - - middle = len(samples) // 2 - median_ms = float(statistics.median(samples)) - first_half_median_ms = float(statistics.median(samples[:middle])) - second_half_median_ms = float(statistics.median(samples[middle:])) - half_change_percent = ( - abs(second_half_median_ms - first_half_median_ms) / first_half_median_ms * 100.0 - ) - samples_within_band = sum( - abs(sample - median_ms) / median_ms * 100.0 <= _TIMING_STABILITY_MEDIAN_BAND_PERCENT - for sample in samples - ) - stable = ( - half_change_percent <= _TIMING_STABILITY_MAX_HALF_CHANGE_PERCENT - and samples_within_band >= _TIMING_STABILITY_MIN_IN_BAND - ) - return { - "status": "stable" if stable else "unstable", - "sample_count": len(samples), - "median_ms": median_ms, - "first_half_median_ms": first_half_median_ms, - "second_half_median_ms": second_half_median_ms, - "half_median_change_percent": half_change_percent, - "samples_within_band": samples_within_band, - } - - -def _measurement_stability( - reference: Mapping[str, Any], candidate: Mapping[str, Any] -) -> dict[str, Any]: - reference_result = _timing_stability(reference.get("samples_ms", [])) - candidate_result = _timing_stability(candidate.get("samples_ms", [])) - statuses = {reference_result["status"], candidate_result["status"]} - if statuses == {"stable"}: - status = "stable" - elif "unstable" in statuses: - status = "unstable" - else: - status = "not_evaluated" - return { - "status": status, - "policy": { - "required_samples": _TIMING_STABILITY_SAMPLE_COUNT, - "max_half_median_change_percent": _TIMING_STABILITY_MAX_HALF_CHANGE_PERCENT, - "median_band_percent": _TIMING_STABILITY_MEDIAN_BAND_PERCENT, - "minimum_samples_within_band": _TIMING_STABILITY_MIN_IN_BAND, - }, - "reference": reference_result, - "candidate": candidate_result, - } - - def compare( entry: ResolvedEntry, candidate: Mapping[str, Any], @@ -1309,8 +1241,37 @@ def _output_contract( "maximum": limit, }, ) + if contract == "vision-language-text": + left_text = _normalized_text(left.get("text")).strip(".,!?;:'\"") + right_text = _normalized_text(right.get("text")).strip(".,!?;:'\"") + distance = _text_distance(left_text, right_text) + limit = float(entry.spec["baseline"].get("max_normalized_edit_distance", 0.15)) + return ( + bool(left_text) and distance <= limit, + "vision-language text distance exceeds the contract", + { + "normalized_edit_distance": distance, + "maximum": limit, + }, + ) if contract == "localization": return _localization_contract(entry, left, right) + if contract == "metric-geometry-parity": + try: + metrics = metric_geometry_metrics(left, right) + passed = metric_geometry_passes(metrics, entry.spec["baseline"]) + except (OSError, ValueError) as error: + return ( + False, + f"metric-geometry parity could not be evaluated: {error}", + {"contract": contract, "numerical_parity_checked": True}, + ) + evidence = { + "contract": contract, + "numerical_parity_checked": True, + **metrics, + } + return passed, "metric-geometry parity is outside the contract" if not passed else "", evidence if contract in {"metric-geometry-shape", "molecular-structure-shape", "pose-refinement-shape"}: signature = { "metric-geometry-shape": _metric_geometry_signature, @@ -1351,6 +1312,12 @@ def _output_contract( right_shape = tuple(right.get(name) for name in ("num_masks", "height", "width")) matched = None not in left_shape and left_shape == right_shape return matched, "segmentation output shape differs" if not matched else "", None + if contract == "prompted-mask-parity": + return _prompted_mask_parity(entry, left, right) + if contract == "instance-mask-parity": + return _instance_mask_parity(entry, left, right) + if contract == "detection-parity": + return _detection_parity(entry, left, right) if contract == "classification-top-class": left_class = left.get("top_class") right_class = right.get("top_class") @@ -1393,6 +1360,23 @@ def _output_contract( and right.get("finite") is True ) return matched, "robot action output contract differs" if not matched else "", None + if contract == "robot-action-parity": + try: + metrics = robot_action_metrics(left, right) + passed = robot_action_passes(metrics, entry.spec["baseline"]) + except ValueError as error: + return ( + False, + f"robot action parity could not be evaluated: {error}", + {"contract": contract, "numerical_parity_checked": True}, + ) + evidence = { + "contract": contract, + "numerical_parity_checked": True, + **metrics, + } + reason = "robot action numerical parity is outside the contract" if not passed else "" + return passed, reason, evidence if contract == "disparity-parity": evidence = _disparity(entry, left, right) return bool(evidence["passed"]), str(evidence.get("reason", "")), evidence @@ -1978,6 +1962,269 @@ def _localization_contract( return distance <= limit, "localization text distance exceeds the contract", evidence +def _detection_values(summary: Mapping[str, Any]) -> list[dict[str, Any]] | None: + boxes = summary.get("boxes") + scores = summary.get("scores") + classes = summary.get("class_ids", summary.get("classes")) + if not isinstance(boxes, list) or not isinstance(scores, list) or not isinstance(classes, list): + return None + if boxes and isinstance(boxes[0], list): + rows = boxes + else: + if len(boxes) % 4: + return None + rows = [boxes[index : index + 4] for index in range(0, len(boxes), 4)] + if len(rows) != len(scores) or len(rows) != len(classes): + return None + detections = [] + for box, score, class_id in zip(rows, scores, classes, strict=True): + if ( + not isinstance(box, list) + or len(box) != 4 + or isinstance(class_id, bool) + or not isinstance(class_id, int) + ): + return None + try: + coordinates = tuple(float(value) for value in box) + confidence = float(score) + except (TypeError, ValueError): + return None + if not all(math.isfinite(value) for value in (*coordinates, confidence)): + return None + detections.append( + {"box": coordinates, "score": confidence, "class_id": class_id} + ) + return detections + + +def _prompted_mask_values( + summary: Mapping[str, Any], +) -> tuple[int, int, list[list[bool]]] | None: + height = summary.get("height") + width = summary.get("width") + count = summary.get("num_masks") + masks = summary.get("masks") + kind = summary.get("mask_kind", "logits") + if ( + isinstance(height, bool) + or not isinstance(height, int) + or height < 1 + or isinstance(width, bool) + or not isinstance(width, int) + or width < 1 + or isinstance(count, bool) + or not isinstance(count, int) + or count < 1 + or not isinstance(masks, list) + or len(masks) != count * height * width + or kind not in {"logits", "probability", "binary"} + ): + return None + if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in masks): + return None + numeric = [float(value) for value in masks] + if not all(math.isfinite(value) for value in numeric): + return None + threshold = 0.0 if kind == "logits" else 0.5 + area = height * width + return height, width, [ + [value > threshold for value in numeric[index : index + area]] + for index in range(0, len(numeric), area) + ] + + +def _binary_mask_iou(left: Sequence[bool], right: Sequence[bool]) -> float: + intersection = sum(a and b for a, b in zip(left, right, strict=True)) + union = sum(a or b for a, b in zip(left, right, strict=True)) + return intersection / union if union else 1.0 + + +def _prompted_mask_parity( + entry: ResolvedEntry, + left: Mapping[str, Any], + right: Mapping[str, Any], +) -> tuple[bool, str, dict[str, Any] | None]: + candidate = _prompted_mask_values(left) + reference = _prompted_mask_values(right) + if candidate is None or reference is None: + return False, "prompted mask output is invalid", None + if candidate[:2] != reference[:2] or len(candidate[2]) != len(reference[2]): + return ( + False, + "prompted mask shape or count differs", + { + "candidate_shape": [len(candidate[2]), *candidate[:2]], + "reference_shape": [len(reference[2]), *reference[:2]], + }, + ) + matched_reference: set[int] = set() + matches = [] + for mask in candidate[2]: + choices = [ + (_binary_mask_iou(mask, expected), index) + for index, expected in enumerate(reference[2]) + if index not in matched_reference + ] + iou, index = max(choices) + matched_reference.add(index) + matches.append(iou) + minimum_iou = min(matches) + required_iou = float(entry.spec["baseline"].get("min_mask_iou", 0.7)) + evidence = { + "masks": len(matches), + "minimum_mask_iou": minimum_iou, + "required_mask_iou": required_iou, + } + if minimum_iou < required_iou: + return False, "prompted mask IoU is below the contract", evidence + return True, "", evidence + + +def _instance_mask_values( + summary: Mapping[str, Any], +) -> tuple[int, int, list[list[bool]], list[float], list[list[float]]] | None: + masks = _prompted_mask_values(summary) + if masks is None: + return None + scores = summary.get("iou_scores") + boxes = summary.get("boxes") + if ( + not isinstance(scores, list) + or len(scores) != len(masks[2]) + or not isinstance(boxes, list) + or len(boxes) != len(masks[2]) + or summary.get("box_coordinates") != "original_image_pixels_xyxy" + ): + return None + try: + numeric_scores = [float(score) for score in scores] + numeric_boxes = [ + [float(coordinate) for coordinate in box] + for box in boxes + if isinstance(box, list) and len(box) == 4 + ] + except (TypeError, ValueError): + return None + if len(numeric_boxes) != len(boxes) or not all( + math.isfinite(number) + for number in (*numeric_scores, *(value for box in numeric_boxes for value in box)) + ): + return None + return *masks, numeric_scores, numeric_boxes + + +def _instance_mask_parity( + entry: ResolvedEntry, + left: Mapping[str, Any], + right: Mapping[str, Any], +) -> tuple[bool, str, dict[str, Any] | None]: + candidate = _instance_mask_values(left) + reference = _instance_mask_values(right) + if candidate is None or reference is None: + return False, "instance mask output is invalid", None + if candidate[:2] != reference[:2] or len(candidate[2]) != len(reference[2]): + return ( + False, + "instance mask shape or count differs", + { + "candidate_shape": [len(candidate[2]), *candidate[:2]], + "reference_shape": [len(reference[2]), *reference[:2]], + }, + ) + matched_reference: set[int] = set() + matches = [] + for index, mask in enumerate(candidate[2]): + choices = [ + (_binary_mask_iou(mask, expected), expected_index) + for expected_index, expected in enumerate(reference[2]) + if expected_index not in matched_reference + ] + mask_iou, expected_index = max(choices) + matched_reference.add(expected_index) + matches.append( + ( + mask_iou, + _box_iou(candidate[4][index], reference[4][expected_index]), + abs(candidate[3][index] - reference[3][expected_index]), + ) + ) + minimum_mask_iou = min(match[0] for match in matches) + minimum_box_iou = min(match[1] for match in matches) + maximum_score_error = max(match[2] for match in matches) + required_mask_iou = float(entry.spec["baseline"].get("min_mask_iou", 0.7)) + required_box_iou = float(entry.spec["baseline"].get("min_box_iou", 0.9)) + allowed_score_error = float(entry.spec["baseline"].get("max_score_abs_error", 0.05)) + evidence = { + "instances": len(matches), + "minimum_mask_iou": minimum_mask_iou, + "required_mask_iou": required_mask_iou, + "minimum_box_iou": minimum_box_iou, + "required_box_iou": required_box_iou, + "maximum_score_abs_error": maximum_score_error, + "allowed_score_abs_error": allowed_score_error, + } + if minimum_mask_iou < required_mask_iou: + return False, "instance mask IoU is below the contract", evidence + if minimum_box_iou < required_box_iou: + return False, "instance box IoU is below the contract", evidence + if maximum_score_error > allowed_score_error: + return False, "instance score error exceeds the contract", evidence + return True, "", evidence + + +def _detection_parity( + entry: ResolvedEntry, + left: Mapping[str, Any], + right: Mapping[str, Any], +) -> tuple[bool, str, dict[str, Any] | None]: + candidate = _detection_values(left) + reference = _detection_values(right) + if candidate is None or reference is None: + return False, "detection output is invalid", None + if len(candidate) != len(reference): + return ( + False, + "detection count differs", + {"candidate_count": len(candidate), "reference_count": len(reference)}, + ) + matched_reference: set[int] = set() + matches = [] + for detection in sorted(candidate, key=lambda item: -float(item["score"])): + choices = [ + (_box_iou(detection["box"], expected["box"]), index) + for index, expected in enumerate(reference) + if index not in matched_reference + and int(detection["class_id"]) == int(expected["class_id"]) + ] + if not choices: + return False, "detection classes differ", None + iou, index = max(choices) + matched_reference.add(index) + matches.append( + ( + iou, + abs(float(detection["score"]) - float(reference[index]["score"])), + ) + ) + minimum_iou = min((match[0] for match in matches), default=1.0) + maximum_score_error = max((match[1] for match in matches), default=0.0) + required_iou = float(entry.spec["baseline"].get("min_box_iou", 0.5)) + allowed_score_error = float(entry.spec["baseline"].get("max_score_abs_error", 0.05)) + evidence = { + "detections": len(matches), + "minimum_box_iou": minimum_iou, + "required_box_iou": required_iou, + "maximum_score_abs_error": maximum_score_error, + "allowed_score_abs_error": allowed_score_error, + } + if minimum_iou < required_iou: + return False, "detection box IoU is below the contract", evidence + if maximum_score_error > allowed_score_error: + return False, "detection score error exceeds the contract", evidence + return True, "", evidence + + def _box_iou(left: tuple[float, ...], right: tuple[float, ...]) -> float: lx1, ly1, lx2, ly2 = left rx1, ry1, rx2, ry2 = right @@ -2045,7 +2292,7 @@ def _entry_slug(entry_id: str) -> str: return "entry" if value in {"", ".", ".."} else value -def _execute_entry( +def execute_entry( entry: ResolvedEntry, environment: Environment, run_directory: Path, @@ -2346,7 +2593,7 @@ def _run_rows( while (artifact_root / f"attempt-{attempt}").exists(): attempt += 1 try: - row = _execute_entry( + row = execute_entry( entry, environment, run_directory, @@ -2383,180 +2630,6 @@ def _run_rows( return 0 if results["status"] == "completed" else 1 -def write_report( - run_directory: Path, - results: Mapping[str, Any], - preparation: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - source_schema = results.get("schema_version", RESULT_SCHEMA) - row_key = "cases" if source_schema == "trtmc.perf-matrix/v1" else "rows" - rows = [dict(row) for row in results.get(row_key, []) if isinstance(row, Mapping)] - selected_ids = results.get("selected_entry_ids") - if not isinstance(selected_ids, list) or not all( - isinstance(value, str) for value in selected_ids - ): - raise PerfMatrixError("matrix results selected_entry_ids must be strings") - completed_ids = {str(row.get("id")) for row in rows} - counts = { - status: sum(row.get("status") == status for row in rows) - for status in ("green", "yellow", "red", "contract-mismatch", "white") - } - report = { - "schema_version": REPORT_SCHEMA, - "source_schema_version": source_schema, - "generated_at": _now(), - "status": results.get("status", "unknown"), - "suite": results.get("suite"), - "environment": results.get("environment"), - "summary": { - "selected": len(selected_ids), - "pending": sum(entry_id not in completed_ids for entry_id in selected_ids), - "comparable": counts["green"] + counts["yellow"] + counts["red"], - **counts, - }, - "rows": rows, - } - if preparation is not None: - report["preparation"] = dict(preparation) - _write_json(run_directory / "report.json", report) - (run_directory / "report.html").write_text(_report_html(report), encoding="utf-8") - return report - - -def _measurement_html(value: Any) -> str: - if not isinstance(value, Mapping): - return "Not recorded" - metrics = value.get("metrics", {}) - latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} - samples = value.get("samples_ms", []) - measured = ( - sorted( - float(sample) - for sample in samples - if isinstance(sample, (int, float)) - and not isinstance(sample, bool) - and math.isfinite(sample) - and sample > 0 - ) - if isinstance(samples, list) - else [] - ) - parts = [] - for name, percentile in (("p50", 0.5), ("p95", 0.95)): - number = latency.get(name) if isinstance(latency, Mapping) else None - if number is None and measured: - position = (len(measured) - 1) * percentile - lower, upper = math.floor(position), math.ceil(position) - number = measured[lower] + (measured[upper] - measured[lower]) * (position - lower) - if ( - isinstance(number, (int, float)) - and not isinstance(number, bool) - and math.isfinite(number) - ): - parts.append(f"{name}: {number:,.3f} ms") - policy = value.get("measurement_policy", {}) - scope = value.get("timing_scope") or ( - policy.get("timing_scope") if isinstance(policy, Mapping) else None - ) - parts.append("Scope: " + html.escape(str(scope or "not recorded"))) - if isinstance(metrics, Mapping): - parts.extend( - f"{html.escape(name)}: {number:,.3f}" - for name, number in metrics.items() - if name.endswith("_per_s") - and isinstance(number, (int, float)) - and not isinstance(number, bool) - and math.isfinite(number) - ) - return "
".join(parts) - - -def _report_html(report: Mapping[str, Any]) -> str: - rows = [] - for row in report.get("rows", []): - comparison = row.get("comparison", {}) if isinstance(row, Mapping) else {} - measurement_stability = ( - row.get("measurement_stability", {}) if isinstance(row, Mapping) else {} - ) - candidate = _measurement_html( - row.get("candidate") - or {"metrics": {"latency_ms": {"p50": comparison.get("candidate_p50_ms")}}} - ) - reference = _measurement_html( - row.get("reference", row.get("baseline")) - or {"metrics": {"latency_ms": {"p50": comparison.get("reference_p50_ms")}}} - ) - reason = comparison.get("reason", row.get("error", "")) - stability = ( - measurement_stability.get("status", "") - if isinstance(measurement_stability, Mapping) - else "" - ) - evidence = { - key: row[key] - for key in ( - "commands", - "comparison", - "measurement_stability", - "resolved_settings", - "baseline_contract", - ) - if key in row - } - detail = html.escape(json.dumps(evidence, indent=2, sort_keys=True)) - link = "" - artifact = row.get("artifact_dir") - if ( - isinstance(artifact, str) - and artifact - and not Path(artifact).is_absolute() - and ".." not in Path(artifact).parts - and not any(value in artifact for value in (":", "\\")) - ): - from urllib.parse import quote - - link = f'Case artifacts' - rows.append( - "" - f"{html.escape(str(row.get('id', '')))}" - f"{html.escape(str(row.get('model', '')))}" - f"{html.escape(str(row.get('operation', '')))}" - f"{html.escape(str(row.get('status', '')))}" - f"{html.escape(str(stability))}" - f"{candidate}" - f"{reference}" - f"{html.escape(str(reason))}" - f"

Evidence / commands{link}
{detail}
" - "" - ) - return f""" -TRTMC performance - -

TRTMC performance matrix

-

Source data: {html.escape(str(report.get("source_schema_version", RESULT_SCHEMA)))}. Original timing scopes and recorded outcomes are preserved.

-

Status: {html.escape(str(report.get("status", "unknown")))}; -selected: {html.escape(str(report.get("summary", {}).get("selected", 0)))}; -pending: {html.escape(str(report.get("summary", {}).get("pending", 0)))}

- - - -{"".join(rows)}
EntryModelOperationStatusStabilityCandidate measurementsReference measurementsReasonEvidence
-

Green is faster by more than the margin, yellow is within the margin, red is -slower by more than the margin, and white is not comparable.

-

Output comparison follows the suite's declared contract; matching dimensions or token counts alone does not establish model correctness. Use family correctness tests for acceptance.

-

Original results · Report data

- - -""" - def prepare_entries( entries: Sequence[ResolvedEntry], diff --git a/qualification_tests/benchmark_qualification/performance/qualification.py b/qualification_tests/benchmark_qualification/performance/qualification.py new file mode 100644 index 0000000000..c886701616 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/qualification.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Internal Performance qualification through installed trtmc-bench.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Mapping + +from qualification_tests.benchmark_qualification.performance import matrix as performance_matrix +from qualification_tests.benchmark_qualification.performance.runner import run_case + +from ..catalog import QualificationCase, QualificationError, load_benchmark +from ..runtime import ( + RuntimeContext, + reference_environment_options, + reference_python, + require_candidate, + write_model_descriptor, + write_result, +) + + +_COMPLETED_COMPARISONS = frozenset({"green", "yellow", "red"}) + + +def _qualification_status( + process_returncode: int, + matrix: Mapping[str, Any], + row: Mapping[str, Any], +) -> tuple[str, str | None]: + row_error = row.get("error") + if isinstance(row_error, str) and row_error: + return "error", row_error + row_status = row.get("status") + matrix_status = matrix.get("status") + if row_status == "contract-mismatch": + return "failed", None + if ( + row_status in _COMPLETED_COMPARISONS + and matrix_status == "completed" + and process_returncode == 0 + ): + return "passed", None + comparison = row.get("comparison") + reason = comparison.get("reason") if isinstance(comparison, Mapping) else None + if not isinstance(reason, str) or not reason: + reason = ( + f"Performance matrix ended with row status {row_status!r}, " + f"matrix status {matrix_status!r}, and exit code {process_returncode}" + ) + return "error", reason + + +def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[str, Any]: + output = context.case_artifacts(case) + output.mkdir(parents=True, exist_ok=True) + definition = load_benchmark(context.repository, case) + worker, runtime_root = require_candidate(context) + configured = case.values + request = configured.get("request") + baseline = configured.get("reference") + measurement = configured.get("measurement") + reference_timing = definition.get("reference_timing") + if not all( + isinstance(value, Mapping) for value in (request, baseline, measurement, reference_timing) + ): + raise QualificationError( + "Performance request, reference, measurement, and reference timing must be objects" + ) + assert isinstance(request, Mapping) + assert isinstance(baseline, Mapping) + assert isinstance(measurement, Mapping) + assert isinstance(reference_timing, Mapping) + stability = definition.get("stability", {}) + required_samples = stability.get("samples") if isinstance(stability, Mapping) else None + if ( + isinstance(required_samples, bool) + or not isinstance(required_samples, int) + or int(measurement.get("iterations", 10)) < required_samples + ): + raise QualificationError( + "Performance measurement.iterations must satisfy the stability sample count" + ) + request = _resolve_family_assets(case, request) + baseline = _resolve_reference_assets(case, context, baseline) + descriptor = write_model_descriptor(case, output, request, context=context) + entry_id = f"qualification.{case.family}.{case.name}" + spec = { + "id": entry_id, + "family": case.family, + "operation": str(configured["operation"]), + "model": case.model, + "manifest": str(descriptor), + "workload": { + "testcase": case.name, + "request": dict(request), + }, + "measurement": { + "warmup": int(measurement.get("warmup", 5)), + "iterations": int(measurement.get("iterations", 10)), + }, + "baseline": {**dict(baseline), **dict(reference_timing)}, + "equivalence_margin_percent": float( + configured.get("equivalence_margin_percent", 5.0) + ), + } + if stability != { + "samples": 10, + "max_half_median_change_percent": 5.0, + "median_band_percent": 5.0, + "minimum_samples_within_band": 8, + "retries": 1, + }: + raise QualificationError("unsupported Performance stability definition") + results_root = output / "matrix" + environment = performance_matrix.Environment( + name="qualification", + trtmc_bench=context.trtmc_bench, + worker=worker, + hf_runner=context.repository / "qualification_tests/benchmark_qualification/performance/references/hf_transformers.py", + task_runner=context.repository / "qualification_tests/benchmark_qualification/performance/references/generic_reference.py", + reference_python=reference_python(case, context), + results_root=results_root, + scratch_root=output / "scratch", + bundle_cache=context.bundle_cache, + bundle_roots=context.bundle_roots, + runtime_root=runtime_root, + bundle_retention="retain", + local_files_only=os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", + timeout_seconds=int(configured.get("timeout_seconds", 7200)), + references={}, + hf_cache_mode="shared", + hf_cache_retention="retain", + ) + try: + row = run_case( + spec, + environment, + results_root, + no_build=context.no_build, + verbose=context.verbose, + ) + except (OSError, performance_matrix.PerfMatrixError) as error: + raise QualificationError(f"Performance execution failed: {error}") from error + matrix_status = "completed" if row.get("status") in _COMPLETED_COMPARISONS else "failed" + status, error = _qualification_status( + 0 if matrix_status == "completed" else 1, + {"status": matrix_status}, + row, + ) + comparison_value = row.get("comparison") + comparison = comparison_value if isinstance(comparison_value, Mapping) else {} + result = { + "schema_version": "trtmc.qualification-result/v1", + "case": case.id, + "kind": "performance", + "status": status, + "model": case.model, + "benchmark": case.benchmark, + "matrix_run": str(results_root), + "comparison_status": row.get("status") if isinstance(row, Mapping) else None, + "metrics": { + "candidate_p50_ms": comparison.get("candidate_p50_ms"), + "reference_p50_ms": comparison.get("reference_p50_ms"), + "reference_over_candidate_p50": comparison.get("reference_over_candidate_p50"), + }, + "reference_attempts": row.get("reference_attempts", []) if isinstance(row, Mapping) else [], + } + if error is not None: + result["error"] = error + write_result(output, result) + return result + + +def _resolve_family_assets(case: QualificationCase, request: Mapping[str, Any]) -> dict[str, Any]: + resolved = dict(request) + family_root = case.source.parents[2].resolve() + for key, value in request.items(): + if not key.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if path.is_absolute(): + continue + path = (case.source.parent / path).resolve() + if family_root not in path.parents or not path.is_file(): + raise QualificationError( + f"Performance request asset {key!r} is unavailable inside {family_root}: {path}" + ) + if key == "prompt_path": + payload = ( + json.loads(path.read_text(encoding="utf-8")) if path.suffix == ".json" else None + ) + prompt = ( + payload.get("prompt") + if isinstance(payload, Mapping) + else path.read_text(encoding="utf-8").strip() + ) + if not isinstance(prompt, str) or not prompt: + raise QualificationError(f"Performance prompt is unavailable: {path}") + resolved.pop(key, None) + resolved["prompt"] = prompt + else: + resolved[key] = str(path) + return resolved + + +def _resolve_reference_assets( + case: QualificationCase, + context: RuntimeContext, + reference: Mapping[str, Any], +) -> dict[str, Any]: + resolved = dict(reference) + options = reference.get("adapter_options", {}) + if not isinstance(options, Mapping): + raise QualificationError("Performance reference.adapter_options must be an object") + resolved_options = dict(options) + family_root = case.source.parents[2].resolve() + for key, value in options.items(): + if not key.endswith("_path") or not isinstance(value, str): + continue + path = Path(value) + if not path.is_absolute(): + path = (case.source.parent / path).resolve() + if not path.is_relative_to(family_root) or not path.is_file(): + raise QualificationError(f"Performance reference asset {key!r} is unavailable: {path}") + resolved_options[key] = str(path) + resolved["adapter_options"] = reference_environment_options(case, context, resolved_options) + return resolved diff --git a/qualification_tests/benchmark_qualification/performance/reference_harness.py b/qualification_tests/benchmark_qualification/performance/reference_harness.py new file mode 100644 index 0000000000..48d0012faf --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/reference_harness.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic execution and reporting for family-owned Performance references.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import math +from pathlib import Path +import platform +import statistics +import time +from typing import Any, Callable, Mapping, Sequence + +from .references.hf_transformers import flatten_config +from .references.timing_contracts import timing_contract + + +@dataclass(frozen=True) +class Session: + """A loaded reference and the repeatable operation to measure.""" + + invoke: Callable[[], Mapping[str, Any]] + backend: str + timing_scope: str = "task-model-call-wall" + input_preparation_included: bool = False + asset_loading_included: bool = False + materialize: Callable[[dict[str, Any], Path], None] | None = None + + +@dataclass(frozen=True) +class Measurement: + """Already measured family output using the shared result contract.""" + + samples_ms: Sequence[float] + output_summary: Mapping[str, Any] + backend: str + timing_scope: str = "task-model-call-wall" + input_preparation_included: bool = False + asset_loading_included: bool = False + + +def parser(description: str) -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=description) + value.add_argument("--family", required=True) + value.add_argument("--operation", required=True) + value.add_argument("--manifest", required=True, type=Path) + value.add_argument("--adapter-options-json", default="{}") + value.add_argument("--timing-contract-json", required=True) + value.add_argument("--padding", default="longest") + value.add_argument("--model", required=True) + value.add_argument("--revision") + value.add_argument("--request-json", required=True) + value.add_argument("--precision", required=True, choices=("fp16", "bf16", "fp32")) + value.add_argument("--mode", required=True) + value.add_argument("--warmup", required=True, type=int) + value.add_argument("--iterations", required=True, type=int) + value.add_argument("--case-name", required=True) + value.add_argument("--testcase-name") + value.add_argument("--output", required=True, type=Path) + value.add_argument("--selected-task", required=True) + value.add_argument("--trust-remote-code", action="store_true") + value.add_argument("--local-files-only", action="store_true") + return value + + +def json_object(raw: str, label: str) -> dict[str, Any]: + try: + value = json.loads(raw) + except json.JSONDecodeError as error: + raise ValueError(f"{label} is not valid JSON: {error}") from error + if not isinstance(value, dict): + raise ValueError(f"{label} must be an object") + return value + + +def synchronize() -> None: + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.synchronize() + except ImportError: + pass + + +def measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: + if warmup < 0 or iterations < 1: + raise ValueError("warmup must be non-negative and iterations must be positive") + output: Mapping[str, Any] = {} + for _ in range(warmup): + output = session.invoke() + synchronize() + samples = [] + for _ in range(iterations): + synchronize() + started = time.perf_counter() + output = session.invoke() + synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + if not all(math.isfinite(value) and value > 0.0 for value in samples): + raise RuntimeError("reference produced an invalid timing sample") + return samples, dict(output) + + +def environment() -> dict[str, Any]: + value: dict[str, Any] = { + "platform": platform.platform(), + "python": platform.python_version(), + } + try: + import torch + + value["torch"] = str(torch.__version__) + if torch.cuda.is_available(): + value["gpu"] = torch.cuda.get_device_name(0) + value["cuda"] = torch.version.cuda + except ImportError: + pass + return value + + +def run( + argv: Sequence[str], + *, + description: str, + load: Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session], +) -> int: + arguments = parser(description).parse_args(argv) + request = flatten_config(json_object(arguments.request_json, "--request-json")) + options = json_object(arguments.adapter_options_json, "--adapter-options-json") + load_started = time.perf_counter() + session = load(arguments, request, options) + load_seconds = time.perf_counter() - load_started + samples, output_summary = measure(session, arguments.warmup, arguments.iterations) + if session.materialize is not None: + session.materialize(output_summary, arguments.output) + return _write( + arguments, + Measurement( + samples, + output_summary, + session.backend, + session.timing_scope, + session.input_preparation_included, + session.asset_loading_included, + ), + load_seconds=load_seconds, + ) + + +def run_premeasured( + argv: Sequence[str], + *, + description: str, + execute: Callable[ + [argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Measurement + ], +) -> int: + arguments = parser(description).parse_args(argv) + request = flatten_config(json_object(arguments.request_json, "--request-json")) + options = json_object(arguments.adapter_options_json, "--adapter-options-json") + measurement = execute(arguments, request, options) + return _write(arguments, measurement, load_seconds=None) + + +def _write( + arguments: argparse.Namespace, + measurement: Measurement, + *, + load_seconds: float | None, +) -> int: + samples = [float(value) for value in measurement.samples_ms] + if len(samples) != arguments.iterations or not all( + math.isfinite(value) and value > 0.0 for value in samples + ): + raise RuntimeError("reference must return one finite positive sample per iteration") + actual = { + "timing_scope": measurement.timing_scope, + "input_preparation_included": measurement.input_preparation_included, + "asset_loading_included": measurement.asset_loading_included, + } + declared = json_object(arguments.timing_contract_json, "--timing-contract-json") + expected = timing_contract(runner="task-reference", declared=declared) if declared else None + if expected is not None and any(actual[name] != expected[name] for name in actual): + raise RuntimeError(f"family reference timing drifted: actual={actual}, declared={expected}") + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": measurement.backend, + "framework": measurement.backend, + "family": arguments.family, + "operation": arguments.operation, + "selected_task": arguments.selected_task, + "mode": arguments.mode, + "precision": arguments.precision, + "padding": arguments.padding, + "model": arguments.model, + "case_name": arguments.case_name, + "model_load_included": False, + "model_load_seconds": load_seconds, + **actual, + "measurement": {"warmup": arguments.warmup, "iterations": arguments.iterations}, + "measurement_policy": { + **actual, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": statistics.median(samples), + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": dict(measurement.output_summary), + "environment": environment(), + "finished_at": datetime.now(timezone.utc).isoformat(), + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 diff --git a/qualification_tests/benchmark_qualification/performance/reference_protocol.py b/qualification_tests/benchmark_qualification/performance/reference_protocol.py new file mode 100644 index 0000000000..e2b7db98fd --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/reference_protocol.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command protocol shared by Accuracy and Performance reference execution.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + + +def command( + *, + python: Path, + generic_runner: Path, + script: Path | None, + family: str, + operation: str, + manifest: Path, + selected_task: str | None, + testcase_name: str | None, + adapter: str | None, + adapter_options: Mapping[str, Any], + timing_contract: Mapping[str, Any], + padding: str, + model: str, + revision: str | None, + request: Mapping[str, Any], + precision: str, + mode: str, + warmup: int, + iterations: int, + case_name: str, + output: Path, + trust_remote_code: bool, + local_files_only: bool, +) -> list[str]: + if (script is None) == (adapter is None): + raise ValueError("reference requires exactly one of script or adapter") + arguments = [ + str(python), + str(script or generic_runner), + ] + if script is not None: + arguments.extend(("--family", family)) + else: + arguments.extend(("--adapter", str(adapter))) + arguments.extend( + ( + "--operation", + operation, + "--manifest", + str(manifest), + "--adapter-options-json", + json.dumps(dict(adapter_options), ensure_ascii=True, separators=(",", ":")), + "--timing-contract-json", + json.dumps(dict(timing_contract), ensure_ascii=True, separators=(",", ":")), + "--padding", + padding, + "--model", + model, + "--request-json", + json.dumps(dict(request), ensure_ascii=True, separators=(",", ":")), + "--precision", + precision, + "--mode", + mode, + "--warmup", + str(warmup), + "--iterations", + str(iterations), + "--case-name", + case_name, + "--output", + str(output), + ) + ) + if script is None and testcase_name is not None: + arguments.extend(("--testcase-name", testcase_name)) + if selected_task is not None: + arguments.extend(("--selected-task", selected_task)) + if revision: + arguments.extend(("--revision", revision)) + if trust_remote_code: + arguments.append("--trust-remote-code") + if local_files_only: + arguments.append("--local-files-only") + return arguments diff --git a/qualification_tests/benchmark_qualification/performance/references/__init__.py b/qualification_tests/benchmark_qualification/performance/references/__init__.py new file mode 100644 index 0000000000..e25025f300 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/references/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reference implementations used only by internal qualification.""" diff --git a/apps/benchmark/performance/baselines/audio_reference.py b/qualification_tests/benchmark_qualification/performance/references/audio_reference.py similarity index 100% rename from apps/benchmark/performance/baselines/audio_reference.py rename to qualification_tests/benchmark_qualification/performance/references/audio_reference.py diff --git a/qualification_tests/benchmark_qualification/performance/references/generic_reference.py b/qualification_tests/benchmark_qualification/performance/references/generic_reference.py new file mode 100644 index 0000000000..43f75c0015 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/references/generic_reference.py @@ -0,0 +1,1685 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmark task-aligned non-text model reference operations.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import json +import math +import os +from pathlib import Path +import platform +import statistics +import sys +import time +from typing import Any, Callable, Mapping, Sequence + + +REPOSITORY = Path(__file__).resolve().parents[4] +for source_root in ( + REPOSITORY, + REPOSITORY / "core/builder", + REPOSITORY / "apps/benchmark", +): + if str(source_root) not in sys.path: + sys.path.insert(0, str(source_root)) + +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract # noqa: E402 +from qualification_tests.benchmark_qualification.performance.references.hf_transformers import _batch_prompt, flatten_config # noqa: E402 + +ADAPTERS = ( + "hf-diffusers", + "hf-diffusers-modular", + "hf-transformers-asr", + "hf-transformers-embedding", + "hf-transformers-reranking", + "hf-transformers-tts", + "hf-transformers-vision", + "hf-transformers-vlm", + "pytorch-timeseries", + "timm-classification", +) +PYTORCH_ADAPTERS = {"pytorch-timeseries"} + + +@dataclass(frozen=True) +class Session: + """One loaded reference model and its repeatable timed operation.""" + + invoke: Callable[[], Any] + framework: str + timing_scope: str = "task-model-call-wall" + input_preparation_included: bool = False + asset_loading_included: bool = False + compile_evidence: dict[str, Any] | None = None + summarize: Callable[[Any], Mapping[str, Any]] | None = None + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adapter", required=True, choices=ADAPTERS) + parser.add_argument("--operation", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--revision") + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument( + "--selected-task", help="Semantic Task selected independently of bundle identity" + ) + parser.add_argument("--request-json", required=True) + parser.add_argument("--adapter-options-json", default="{}") + parser.add_argument("--timing-contract-json", default="{}") + parser.add_argument("--precision", required=True, choices=("fp16", "fp32", "bf16")) + parser.add_argument( + "--mode", required=True, choices=("hf-eager", "pytorch-eager", "torch-compile") + ) + parser.add_argument("--padding", default="longest") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--warmup", required=True, type=int) + parser.add_argument("--iterations", required=True, type=int) + parser.add_argument("--case-name", required=True) + parser.add_argument( + "--testcase-name", help="Selected manifest testcase, not the performance entry ID" + ) + parser.add_argument("--output", required=True, type=Path) + return parser + + +def _json_object(raw: str, label: str) -> dict[str, Any]: + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{label} is not valid JSON: {exc}") from exc + if not isinstance(value, dict): + raise ValueError(f"{label} must contain an object") + return value + + +def _selected_task(arguments: argparse.Namespace) -> str: + selected = getattr(arguments, "selected_task", None) + if selected is None: + return json.loads(arguments.manifest.read_text(encoding="utf-8"))["task"] + if not isinstance(selected, str) or not selected or selected != selected.strip(): + raise ValueError("selected_task must be a nonempty Task ID without surrounding whitespace") + return selected + + +def _torch_dtype(torch_module: Any, precision: str) -> Any: + return { + "fp16": torch_module.float16, + "fp32": torch_module.float32, + "bf16": torch_module.bfloat16, + }[precision] + + +def _seed_all(torch_module: Any, seed: int) -> None: + import random + + import numpy as np + + random.seed(seed) + np.random.seed(seed) + torch_module.manual_seed(seed) + if torch_module.cuda.is_available(): + torch_module.cuda.manual_seed_all(seed) + + +def _request_seed(request: Mapping[str, Any], default: Any = 42) -> int: + value = request.get("seed", default) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("reference request seed must be an integer") + return value + + +def _bark_generation_options(request: Mapping[str, Any]) -> dict[str, int]: + max_new_tokens = request.get("max_new_tokens", 0) + if isinstance(max_new_tokens, bool) or not isinstance(max_new_tokens, int): + raise ValueError("Bark request max_new_tokens must be an integer") + if max_new_tokens <= 0: + return {} + return {"semantic_max_new_tokens": max_new_tokens} + + +def _load_kwargs(arguments: argparse.Namespace, torch_module: Any) -> dict[str, Any]: + values: dict[str, Any] = { + "trust_remote_code": arguments.trust_remote_code, + "local_files_only": arguments.local_files_only, + "torch_dtype": _torch_dtype(torch_module, arguments.precision), + } + if arguments.revision: + values["revision"] = arguments.revision + return values + + +def _processor_kwargs(arguments: argparse.Namespace) -> dict[str, Any]: + values: dict[str, Any] = { + "trust_remote_code": arguments.trust_remote_code, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + values["revision"] = arguments.revision + return values + + +def _cached_snapshot_path(repo_id: str, requested: str | None, marker_file: str) -> Path | None: + if Path(repo_id).exists(): + return Path(repo_id).resolve() + try: + from huggingface_hub import try_to_load_from_cache + + cached = try_to_load_from_cache( + repo_id=repo_id, + filename=marker_file, + revision=requested or "main", + ) + except (ImportError, OSError, ValueError): + return None + if isinstance(cached, str) and Path(cached).is_file(): + return Path(cached).parent.resolve() + return None + + +def _to_device(value: Any, device: Any, dtype: Any = None) -> Any: + if isinstance(value, Mapping): + return {name: _to_device(item, device, dtype) for name, item in value.items()} + if hasattr(value, "to"): + if dtype is not None and getattr(value, "is_floating_point", lambda: False)(): + return value.to(device=device, dtype=dtype) + return value.to(device) + return value + + +def _asset_path(arguments: argparse.Namespace, request: Mapping[str, Any], key: str) -> Path: + raw = str(request.get(key, "") or "") + if not raw: + raise ValueError(f"{arguments.adapter} reference requires request.{key}") + path = Path(raw) + if not path.is_absolute(): + path = arguments.manifest.resolve().parent.parent / path + if not path.is_file(): + raise FileNotFoundError(f"reference input does not exist: {path}") + return path + + +def _tensor_summary(value: Any) -> dict[str, Any]: + shape = [int(dim) for dim in value.shape] + return { + "shape": shape, + "element_count": int(value.numel()), + "finite": bool(value.isfinite().all().item()), + } + + +def _flatten_tensor_values(value: Any) -> list[float]: + flattened: list[float] = [] + + def visit(item: Any) -> None: + if isinstance(item, (list, tuple)): + for child in item: + visit(child) + else: + flattened.append(float(item)) + + visit(value.detach().float().cpu().tolist()) + return flattened + + +def _forecast_summary( + value: Any, task: str, quantile_levels: Sequence[float] = () +) -> dict[str, Any]: + summary = _tensor_summary(value) + summary["values"] = _flatten_tensor_values(value) + if task not in {"series_to_point_forecast", "series_to_quantile_forecast"}: + return summary + shape = summary["shape"] + if len(shape) not in {2, 3} or shape[0] != 1: + raise ValueError("forecast reference requires an explicit one-series batch axis") + if task == "series_to_point_forecast": + horizon, channels = shape[1], shape[2] if len(shape) == 3 else 1 + summary.update(shape=[horizon, channels], axes=["horizon", "channel"]) + else: + if len(shape) != 3 or len(quantile_levels) != shape[1]: + raise ValueError( + "quantile forecast reference requires exact checkpoint quantile levels" + ) + horizon = shape[2] + summary.update( + shape=[shape[1], horizon, 1], + axes=["quantile", "horizon", "channel"], + quantile_levels=list(quantile_levels), + ) + summary.update( + forecast_elements=summary["element_count"], horizon_steps=list(range(1, horizon + 1)) + ) + return summary + + +def _regression_values_summary(value: Any) -> dict[str, Any]: + summary = _tensor_summary(value) + shape = summary["shape"] + if len(shape) != 2 or shape[0] != 1 or shape[1] <= 0: + raise ValueError("deterministic regression requires one batch with a nonempty target axis") + if not summary["finite"]: + raise ValueError("deterministic regression target values must be finite") + return { + "kind": "regression_values", + "target_count": shape[1], + "regression_targets": shape[1], + "parameter_elements": 0, + "values": value[0].detach().float().cpu().tolist(), + "axes": ["target"], + "target_names": [], + "target_units": [], + } + + +def _regression_summary( + output: Any, distribution: str, parameter_names: Sequence[str] +) -> dict[str, Any]: + if distribution not in {"normal", "student_t", "negative_binomial"}: + raise ValueError("regression reference requires a declared probability distribution") + if not isinstance(output, (tuple, list)) or len(output) != len(parameter_names) or not output: + raise ValueError("regression reference parameters do not match the declared distribution") + # Transformers names distribution parameters loc/df. The public Task + # contract names them location/degrees_of_freedom; never infer tuple order. + aliases = {"loc": "location", "df": "degrees_of_freedom"} + names = [aliases.get(name, name) for name in parameter_names] + required = { + "normal": {"location", "scale"}, + "student_t": {"degrees_of_freedom", "location", "scale"}, + "negative_binomial": {"total_count", "logits"}, + }[distribution] + if len(names) != len(required) or set(names) != required: + raise ValueError( + "regression reference parameter names do not match the declared distribution" + ) + parameters = [] + targets = None + for name, value in zip(names, output): + shape = tuple(int(size) for size in value.shape) + if ( + len(shape) != 2 + or shape[0] != 1 + or shape[1] <= 0 + or (targets is not None and shape[1] != targets) + ): + raise ValueError( + "regression parameters must each have one batch and the same target axis" + ) + targets = shape[1] + if not _tensor_summary(value)["finite"]: + raise ValueError("regression reference returned nonfinite parameters") + parameters.append({"name": name, "values": value[0].detach().float().cpu().tolist()}) + return { + "distribution": distribution, + "target_count": targets, + "regression_targets": targets, + "parameter_elements": targets * len(parameters), + "axes": ["target"], + "parameters": parameters, + "target_names": [], + "target_units": [], + } + + +def _task_value( + arguments: argparse.Namespace, + request: Mapping[str, Any], + name: str, + default: Any = None, +) -> Any: + if name in request: + return request[name] + return default + + +def _load_tts( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + import torch + from transformers import AutoProcessor, BarkModel + + torch.use_deterministic_algorithms(True) + device = torch.device("cuda") + prompt = str(request.get("prompt", "")) + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + BarkModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + inputs = _to_device(processor(prompt, return_tensors="pt"), device) + seed = _request_seed(request, 42) + generation_options = _bark_generation_options(request) + + def invoke() -> Mapping[str, Any]: + _seed_all(torch, seed) + with torch.inference_mode(): + audio = model.generate(**inputs, **generation_options) + return { + "audio_samples": int(audio.numel()), + "sample_rate": int(model.generation_config.sample_rate), + "_audio_f32": audio.detach().float().cpu().reshape(-1).numpy(), + } + + return Session(invoke, "transformers") + + +def _load_asr( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + import torch + from qualification_tests.benchmark_qualification.performance.references.audio_reference import ( + read_wav_float32, + resample_audio, + ) + + audio, sample_rate = read_wav_float32(str(_asset_path(arguments, request, "audio_path"))) + target_rate = 16_000 + audio = resample_audio(audio, sample_rate, target_rate) + max_new_tokens = int(request.get("max_new_tokens", 100)) + device = torch.device("cuda") + + from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + AutoModelForSpeechSeq2Seq.from_pretrained( + arguments.model, **_load_kwargs(arguments, torch) + ) + .eval() + .to(device) + ) + processor_options: dict[str, Any] = { + "sampling_rate": target_rate, + "return_tensors": "pt", + } + language = str(request.get("language", "") or "") + if language: + processor_options["language"] = language + inputs = _to_device( + processor(audio, **processor_options), device, next(model.parameters()).dtype + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + generated = model.generate(**inputs, max_new_tokens=max_new_tokens) + sequences = generated.sequences if hasattr(generated, "sequences") else generated + token_ids = [int(token) for token in sequences[0].detach().cpu().tolist()] + return { + "text": processor.batch_decode(sequences, skip_special_tokens=True)[0], + "token_ids": token_ids, + "output_tokens": len(token_ids), + } + + return Session(invoke, "transformers") + + +def _load_vlm_model( + transformers_module: Any, class_name: str, model_id: str, kwargs: dict[str, Any] +) -> Any: + if class_name not in { + "AutoModel", + "AutoModelForCausalLM", + "AutoModelForImageTextToText", + }: + raise ValueError(f"unsupported vision-language model_class: {class_name}") + model_class = getattr(transformers_module, class_name) + return model_class.from_pretrained(model_id, **kwargs) + + +def _vl_prompt_has_image_placeholder(text: str) -> bool: + return any( + marker in text + for marker in ( + "<|image_pad|>", + "<|vision_start|>", + "<|image_1|>", + "", + "", + ) + ) + + +def _load_vlm( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + workflow = str(options.get("workflow", "chat")) + if workflow not in {"chat", "single-image-token-chat"}: + raise ValueError(f"unsupported vision-language workflow: {workflow}") + + import torch + import transformers + from PIL import Image + from transformers import AutoProcessor + + device = torch.device("cuda") + processor = AutoProcessor.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + load_options = _load_kwargs(arguments, torch) + if bool(options.get("eager_attention", False)): + config = transformers.AutoConfig.from_pretrained( + arguments.model, **_processor_kwargs(arguments) + ) + config._attn_implementation = "eager" + config._attn_implementation_internal = "eager" + load_options.update({"config": config, "attn_implementation": "eager"}) + model = ( + _load_vlm_model( + transformers, + str(options.get("model_class", "AutoModelForImageTextToText")), + arguments.model, + load_options, + ) + .eval() + .to(device) + ) + image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") + prompt = str(request.get("prompt", "")) + max_new_tokens = int(request.get("max_new_tokens", 16)) + + if workflow == "single-image-token-chat": + messages = [{"role": "user", "content": f"<|image_1|>{prompt}"}] + template_owner = getattr(processor, "tokenizer", processor) + else: + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": prompt}, + ], + } + ] + template_owner = processor + rendered = template_owner.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + if not isinstance(rendered, str) or not _vl_prompt_has_image_placeholder(rendered): + raise ValueError("vision-language chat template lost the image placeholder") + inputs = processor(text=rendered, images=image, padding=True, return_tensors="pt") + inputs = _to_device(inputs, device, next(model.parameters()).dtype) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + output_ids = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=False, + num_beams=1, + ) + input_length = int(inputs["input_ids"].shape[-1]) + sequence = output_ids[0] + generated = sequence[input_length:] if sequence.shape[0] > input_length else sequence + token_ids = [int(token) for token in generated.detach().cpu().tolist()] + text = processor.decode(generated, skip_special_tokens=True) + return {"text": text, "token_ids": token_ids, "output_tokens": len(token_ids)} + + return Session(invoke, "transformers") + + +def _load_embedding( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + configured = _json_object( + getattr(arguments, "timing_contract_json", "{}"), "--timing-contract-json" + ) + if not configured: + raise ValueError("embedding reference requires explicit --timing-contract-json") + declared_timing = timing_contract(runner="task-reference", declared=configured) + if declared_timing["asset_loading_included"]: + raise ValueError( + "embedding reference preloads its assets; asset_loading_included must be false" + ) + prompts = _batch_prompt(request) + if len(prompts) != 1: + raise ValueError("embedding reference requires exactly one text input") + import torch + from transformers import AutoModel, AutoTokenizer + + device = torch.device("cuda") + tokenizer = AutoTokenizer.from_pretrained(arguments.model, **_processor_kwargs(arguments)) + model = ( + AutoModel.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + compile_evidence = _compile_forward(model) if arguments.mode == "torch-compile" else None + prompt = prompts[0] + + def prepare_inputs() -> Mapping[str, Any]: + return _to_device( + tokenizer(prompt, return_tensors="pt", truncation=True), + device, + ) + + prepared_inputs = None + if not declared_timing["input_preparation_included"]: + prepared_inputs = prepare_inputs() + + def invoke() -> Any: + inputs = prepare_inputs() if prepared_inputs is None else prepared_inputs + with torch.inference_mode(): + outputs = model(**inputs, output_hidden_states=True) + hidden = getattr(outputs, "last_hidden_state", None) + if hidden is None: + hidden = outputs.hidden_states[-1] + mask = inputs.get("attention_mask", torch.ones(hidden.shape[:2], device=device)) + mask = mask.unsqueeze(-1).to(hidden.dtype) + vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) + vector = torch.nn.functional.normalize(vector, p=2, dim=-1) + return vector.detach().to("cpu", dtype=torch.float32) + + def summarize(vector: Any) -> Mapping[str, Any]: + summary = _tensor_summary(vector) + summary.update( + { + "embedding_vectors": 1, + "embedding_elements": int(vector.numel()), + "dim": int(vector.shape[-1]), + } + ) + return summary + + return Session( + invoke, + "transformers", + timing_scope=str(declared_timing["timing_scope"]), + input_preparation_included=bool(declared_timing["input_preparation_included"]), + asset_loading_included=False, + compile_evidence=compile_evidence, + summarize=summarize, + ) + + +def _load_reranking( + arguments: argparse.Namespace, + request: Mapping[str, Any], + _options: Mapping[str, Any], +) -> Session: + import torch + from transformers import AutoModelForSequenceClassification, AutoProcessor + + device = torch.device("cuda") + documents = [str(document) for document in request.get("documents", [])] + query = str(request.get("query", "")) + model = ( + AutoModelForSequenceClassification.from_pretrained( + arguments.model, **_load_kwargs(arguments, torch) + ) + .eval() + .to(device) + ) + processor = AutoProcessor.from_pretrained( + arguments.model, + max_input_tiles=6, + use_thumbnail=True, + rerank_max_length=8192, + **_processor_kwargs(arguments), + ) + examples = [ + {"question": query, "doc_text": document, "doc_image": ""} for document in documents + ] + inputs = processor.process_queries_documents_crossencoder(examples) + inputs = _to_device(inputs, device) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + logits = model(**inputs).logits.detach().float().cpu() + if logits.ndim == 2: + logits = logits[:, 0] if logits.shape[-1] == 1 else logits[:, -1] + scores = [float(score) for score in logits.reshape(-1).tolist()] + return {"scores": scores, "document_count": len(documents)} + + return Session(invoke, "transformers") + + +_PIXART_TRTMC_MIXED_PRECISION = "pixart_fp16_dit_fp32_t5" + + +def _diffusion_component_precision_contract( + arguments: argparse.Namespace, + options: Mapping[str, Any], +) -> str: + contract = str(options.get("component_precision_contract", "") or "") + if not contract: + return "" + if contract != _PIXART_TRTMC_MIXED_PRECISION: + raise ValueError(f"unsupported diffusion component precision contract: {contract}") + if arguments.precision != "fp16": + raise ValueError(f"{contract} requires precision=fp16") + return contract + + +def _cast_floating_tensors(value: Any, dtype: Any, torch_module: Any) -> Any: + if isinstance(value, torch_module.Tensor): + return value.to(dtype=dtype) if value.is_floating_point() else value + if isinstance(value, tuple): + return tuple(_cast_floating_tensors(item, dtype, torch_module) for item in value) + if isinstance(value, list): + return [_cast_floating_tensors(item, dtype, torch_module) for item in value] + if isinstance(value, dict): + return { + key: _cast_floating_tensors(item, dtype, torch_module) for key, item in value.items() + } + return value + + +def _configure_diffusion_component_precision( + pipeline: Any, + arguments: argparse.Namespace, + options: Mapping[str, Any], + torch_module: Any, +) -> None: + contract = _diffusion_component_precision_contract(arguments, options) + if not contract: + return + + def fp16_transformer_inputs(_module: Any, args: Any, kwargs: Any) -> Any: + return ( + _cast_floating_tensors(args, torch_module.float16, torch_module), + _cast_floating_tensors(kwargs, torch_module.float16, torch_module), + ) + + def fp32_transformer_output(_module: Any, _args: Any, output: Any) -> Any: + return _cast_floating_tensors(output, torch_module.float32, torch_module) + + pipeline.transformer.register_forward_pre_hook(fp16_transformer_inputs, with_kwargs=True) + pipeline.transformer.register_forward_hook(fp32_transformer_output) + + +def _diffusion_pipeline( + arguments: argparse.Namespace, + torch_module: Any, + options: Mapping[str, Any], +) -> Any: + import diffusers + + model_id = str(options.get("model_id", arguments.model)) + requested_revision = ( + str(options.get("model_revision", getattr(arguments, "revision", None) or "")) or None + ) + model_source: str | Path = model_id + if arguments.local_files_only: + cached = _cached_snapshot_path(model_id, requested_revision, "model_index.json") + if cached is None: + raise FileNotFoundError(f"cached Diffusers snapshot is missing: {model_id}") + model_source = cached + + load_options: dict[str, Any] = {} + component_contract = _diffusion_component_precision_contract(arguments, options) + if component_contract == _PIXART_TRTMC_MIXED_PRECISION: + from transformers import T5EncoderModel + + load_options["text_encoder"] = T5EncoderModel.from_pretrained( + model_source, + subfolder="text_encoder", + torch_dtype=torch_module.float32, + **( + {"revision": requested_revision} + if requested_revision and model_source == model_id + else {} + ), + local_files_only=arguments.local_files_only, + ) + vae_class_name = options.get("vae_class") + if vae_class_name is not None: + if not isinstance(vae_class_name, str) or not vae_class_name.startswith("Autoencoder"): + raise ValueError("diffusion adapter_options.vae_class must name an Autoencoder class") + try: + vae_class = getattr(diffusers, vae_class_name) + except AttributeError as error: + raise ValueError(f"unsupported Diffusers VAE class: {vae_class_name}") from error + vae_precision = str(options.get("vae_precision", arguments.precision)) + load_options["vae"] = vae_class.from_pretrained( + model_source, + subfolder="vae", + torch_dtype=_torch_dtype(torch_module, vae_precision), + **( + {"revision": requested_revision} + if requested_revision and model_source == model_id + else {} + ), + local_files_only=arguments.local_files_only, + ) + return diffusers.DiffusionPipeline.from_pretrained( + model_source, + torch_dtype=_torch_dtype(torch_module, arguments.precision), + **load_options, + **( + {"revision": requested_revision} + if requested_revision and model_source == model_id + else {} + ), + trust_remote_code=bool(options.get("trust_remote_code", arguments.trust_remote_code)), + local_files_only=arguments.local_files_only, + ) + + +def _load_diffusers( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + import inspect + import torch + from PIL import Image + + pipeline = _diffusion_pipeline(arguments, torch, options) + _configure_diffusion_component_precision(pipeline, arguments, options, torch) + if bool(options.get("cpu_offload", False)) and hasattr(pipeline, "enable_model_cpu_offload"): + pipeline.enable_model_cpu_offload() + else: + pipeline.to("cuda") + signature = inspect.signature(pipeline.__call__) + accepted = set(signature.parameters) + accepts_extra = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + if "prompt" in request and "prompts" in request: + raise ValueError("reference request must not provide both prompt and prompts") + prompt = request.get("prompts", request.get("prompt", "")) + if "prompts" in request and not isinstance(prompt, list): + raise ValueError("prompts must contain one string per batch item") + if isinstance(prompt, list): + if not prompt or any(not isinstance(value, str) for value in prompt): + raise ValueError("prompt list must contain one string per batch item") + count = len(prompt) + else: + if not isinstance(prompt, str): + raise ValueError("prompt must be a string or a non-empty list of strings") + count = 1 + batch_size = request.get("batch_size", count) + if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + if isinstance(prompt, list): + if count != batch_size: + raise ValueError("prompt list must contain one string per batch item") + prompt_value: str | list[str] = list(prompt) + else: + prompt_value = [prompt] * batch_size if batch_size > 1 else prompt + values: dict[str, Any] = {"prompt": prompt_value} + negative_prompt = str(request.get("negative_prompt", "")) + if ( + negative_prompt + or "negative_prompt" in request + or bool(options.get("always_negative_prompt", False)) + ): + values["negative_prompt"] = negative_prompt + steps = int(request.get("num_steps", -1)) + if steps > 0: + values.update({"num_inference_steps": steps, "step": steps}) + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + height = int( + request.get("height", request.get("video_height", manifest.get("image_height", 0))) + ) + width = int(request.get("width", request.get("video_width", manifest.get("image_width", 0)))) + if height > 0: + values["height"] = height + if width > 0: + values["width"] = width + num_frames = int( + request.get( + "video_num_frames", + _task_value(arguments, request, "num_frames", manifest.get("video_num_frames", 1)), + ) + ) + if num_frames > 0: + values["num_frames"] = num_frames + for name in ("action", "intrinsics"): + value = str(_task_value(arguments, request, name, "") or "") + if value: + values[name] = value + fps = int(_task_value(arguments, request, "fps", 0)) + if fps > 0: + values["fps"] = fps + for name in ("flow_shift", "translation_speed", "rotation_speed_deg"): + value = _task_value(arguments, request, name) + if value is not None: + values[name] = float(value) + cfg_scale = float(request.get("cfg_scale", -1.0)) + guidance_parameter = str(options.get("guidance_parameter", "guidance_scale")) + if guidance_parameter not in {"guidance_scale", "true_cfg_scale"}: + raise ValueError(f"unsupported diffusion guidance_parameter: {guidance_parameter}") + if cfg_scale >= 0 and guidance_parameter != "true_cfg_scale": + values["cfg_scale"] = cfg_scale + if bool(request.get("no_refiner", False)): + values["no_refiner"] = True + guidance = float(request.get("guidance_scale", -1.0)) + if guidance_parameter == "true_cfg_scale": + true_cfg_scale = guidance if guidance >= 0 else cfg_scale + if true_cfg_scale >= 0: + values["true_cfg_scale"] = true_cfg_scale + elif guidance >= 0: + values["guidance_scale"] = guidance + values["output_type"] = "np" + if "image_path" in request and "image_paths" in request: + raise ValueError("reference request must not provide both image_path and image_paths") + image_path = request.get("image_path", "") + if "image_paths" in request: + image_paths = request["image_paths"] + if ( + not isinstance(image_paths, list) + or len(image_paths) != 1 + or not isinstance(image_paths[0], str) + or not image_paths[0] + ): + raise ValueError("this reference supports exactly one conditioning image") + image_path = image_paths[0] + task = getattr(arguments, "selected_task", None) or manifest["task"] + if task in {"images_text_to_image_edit", "image_edit"} and not image_path: + raise ValueError("image edit reference requires one conditioning image") + if image_path: + if "image" not in accepted and not accepts_extra: + raise ValueError("selected reference pipeline does not accept conditioning image") + values["image"] = Image.open( + _asset_path(arguments, {"image_path": image_path}, "image_path") + ).convert("RGB") + call_values = { + name: value for name, value in values.items() if name in accepted or accepts_extra + } + required = [str(name) for name in options.get("required_call_arguments", [])] + missing = [name for name in required if name not in call_values] + if missing: + raise ValueError( + f"{arguments.adapter} reference is missing required call arguments: " + + ", ".join(missing) + ) + seed = int(request.get("seed", 42)) + raw_seeds = request.get("seeds") + if raw_seeds is not None: + if not isinstance(raw_seeds, list) or len(raw_seeds) != batch_size: + raise ValueError("seeds must contain one integer per batch item") + seeds: int | list[int] = [int(value) for value in raw_seeds] + elif batch_size > 1: + seeds = [seed] * batch_size + else: + seeds = seed + if "generator" in accepted or accepts_extra: + if isinstance(seeds, list): + call_values["generator"] = [ + torch.Generator("cuda").manual_seed(value) for value in seeds + ] + else: + call_values["generator"] = torch.Generator("cuda").manual_seed(seeds) + + def invoke() -> Mapping[str, Any]: + if "generator" in call_values: + generators = call_values["generator"] + if isinstance(generators, list): + for generator, value in zip(generators, seeds, strict=True): + generator.manual_seed(value) + else: + generators.manual_seed(seeds) + result = pipeline(**call_values) + media = getattr(result, "images", None) + if media is None: + media = getattr(result, "frames", None) + media_type = str(request.get("media_type", "image")) + return {**_media_summary(media, media_type), "_media": media} + + return Session( + invoke, + "diffusers", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + ) + + +def _load_modular_diffusers( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + """Run a ModularPipeline whose workflow is declared by the family profile.""" + import torch + from diffusers import ModularPipeline + + workflow = str(options.get("workflow", "")) + output_name = str(options.get("output", "videos")) + if not workflow or not output_name: + raise ValueError("modular Diffusers requires adapter_options.workflow and output") + load_options = { + "workflow": workflow, + "local_files_only": arguments.local_files_only, + } + if arguments.revision: + load_options["revision"] = arguments.revision + pipeline = ModularPipeline.from_pretrained(arguments.model, **load_options) + pipeline.load_components( + dtype=_torch_dtype(torch, arguments.precision), + pretrained_model_name_or_path=arguments.model, + local_files_only=arguments.local_files_only, + ) + pipeline = pipeline.to("cuda") + manifest = json.loads(arguments.manifest.read_text(encoding="utf-8")) + seed = int(request.get("seed", 42)) + generator = torch.Generator().manual_seed(seed) + values: dict[str, Any] = { + "prompt": str(request.get("prompt", "")), + "height": int(request.get("height", manifest.get("image_height", 0))), + "width": int(request.get("width", manifest.get("image_width", 0))), + "num_inference_steps": int(request.get("num_steps", 0)), + "generator": generator, + "output": output_name, + "output_type": "np", + } + frames = int(request.get("video_num_frames", manifest.get("video_num_frames", 1))) + if frames > 1: + values["num_frames"] = frames + for source, target in ( + ("negative_prompt", "negative_prompt"), + ("guidance_scale", "guidance_scale"), + ("cfg_scale", "cfg_scale"), + ): + if source in request: + values[target] = request[source] + + def invoke() -> Mapping[str, Any]: + generator.manual_seed(seed) + media = pipeline(**values) + media_type = "video" if frames > 1 else "image" + return {**_media_summary(media, media_type), "_media": media} + + return Session( + invoke, + "diffusers-modular", + timing_scope="task-pipeline-call-wall", + input_preparation_included=True, + ) + + +def _media_count(media: Any, media_type: str) -> int: + """Count image batches or video frames without coercing arrays to bool.""" + if media is None: + return 0 + try: + outer_count = len(media) + except TypeError: + return 1 + if outer_count == 0 or media_type != "video": + return outer_count + try: + return len(media[0]) + except TypeError: + return outer_count + + +def _media_summary(media: Any, media_type: str) -> dict[str, Any]: + """Describe materialized image/video geometry without copying its pixels.""" + count = _media_count(media, media_type) + item = media + try: + item = item[0] + if media_type == "video": + item = item[0] + except (IndexError, KeyError, TypeError): + pass + width = height = channels = None + size = getattr(item, "size", None) + bands = getattr(item, "getbands", None) + if isinstance(size, tuple) and len(size) == 2 and callable(bands): + width, height = (int(value) for value in size) + channels = len(bands()) + else: + shape = tuple(int(value) for value in getattr(item, "shape", ())) + if len(shape) >= 3: + if shape[-1] in {1, 3, 4}: + height, width, channels = shape[-3:] + elif shape[-3] in {1, 3, 4}: + channels, height, width = shape[-3:] + summary = { + "media_type": media_type, + "media_count": count, + "height": height, + "width": width, + "channels": channels, + } + try: + import numpy as np + + array = np.asarray(media) + if np.issubdtype(array.dtype, np.number): + finite = bool(np.isfinite(array).all()) + if not finite: + raise RuntimeError("reference returned non-finite media values") + summary["finite"] = True + except (TypeError, ValueError): + pass + return summary + + +def _media_items(media: Any, media_type: str) -> list[Any]: + """Return image items, or frames from the first generated video.""" + import numpy as np + + if media_type == "video": + if isinstance(media, np.ndarray): + values = media[0] if media.ndim == 5 else media + else: + values = media[0] if isinstance(media, (list, tuple)) and media else media + else: + values = media + if isinstance(values, np.ndarray): + if values.ndim == 3: + return [values] + return [values[index] for index in range(len(values))] + if isinstance(values, (list, tuple)): + return list(values) + return [values] + + +def _write_media_artifacts(summary: dict[str, Any], output: Path) -> None: + """Materialize deterministic sample frames after reference timing completes.""" + import numpy as np + from PIL import Image + + media = summary.pop("_media") + items = _media_items(media, str(summary.get("media_type", "image"))) + declared = int(summary.get("media_count", 0)) + if declared != len(items) or declared < 1: + raise RuntimeError("reference media count differs from its materialized output") + indices = sorted({0, declared // 2, declared - 1}) + directory = output.with_suffix(".media").resolve() + directory.mkdir(parents=True, exist_ok=True) + artifacts = [] + for index in indices: + item = items[index] + values = np.asarray(item) + scale = 255.0 if np.issubdtype(values.dtype, np.floating) else 1.0 + image = ( + item + if isinstance(item, Image.Image) + else Image.fromarray(np.clip(values * scale, 0, 255).astype(np.uint8)) + ) + path = directory / f"{index:06d}.png" + image.convert("RGB").save(path) + artifacts.append(str(path)) + summary["artifact_indices"] = indices + summary["frame_artifacts" if summary.get("media_type") == "video" else "image_artifacts"] = ( + artifacts + ) + + +def _numeric_values(request: Mapping[str, Any], key: str) -> list[float]: + raw = request.get(key) + if not isinstance(raw, list) or not raw: + raise ValueError(f"time-series reference requires non-empty request.{key}") + return [float(value) for value in raw] + + +def _observed_values(request: Mapping[str, Any], count: int) -> list[float]: + raw = request.get("observed_mask") + if raw is None or raw == []: + return [1.0] * count + values = _numeric_values(request, "observed_mask") + if len(values) != count: + raise ValueError("time-series observed_mask must match past_values") + return values + + +def _align(values: Sequence[float], length: int, fill: float) -> list[float]: + result = [fill] * length + count = min(len(values), length) + result[-count:] = values[-count:] + return result + + +def _patchtst_task(config: Any) -> str: + for attribute in ("patchtst_task", "task_type", "problem_type"): + value = str(getattr(config, attribute, "") or "").lower() + if "class" in value: + return "classification" + if "regress" in value: + return "regression" + if "forecast" in value or "predict" in value: + return "forecast" + + architectures = getattr(config, "architectures", []) or [] + if isinstance(architectures, str): + architectures = [architectures] + for architecture in architectures: + value = str(architecture).lower() + if "class" in value: + return "classification" + if "regress" in value: + return "regression" + if "forecast" in value or "predict" in value: + return "forecast" + return "regression" + + +def _load_timeseries( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + import torch + import transformers + + device = torch.device("cuda") + dtype = _torch_dtype(torch, arguments.precision) + task_id = _selected_task(arguments) + reference_type = str(options.get("reference_type", "")) + if reference_type == "chronos-bolt": + from chronos import ChronosBoltPipeline + + chronos_options = _processor_kwargs(arguments) + chronos_options.update({"device_map": str(device), "dtype": dtype}) + model = ChronosBoltPipeline.from_pretrained(arguments.model, **chronos_options) + compile_evidence = None + if arguments.mode == "torch-compile": + compile_evidence = _compile_forward(model.model) + raw = _numeric_values(request, "past_values") + observed = _observed_values(request, len(raw)) + context = torch.tensor( + [value if mask > 0 else float("nan") for value, mask in zip(raw, observed)], + dtype=dtype, + device=device, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + value = model.predict( + context, + prediction_length=model.model_prediction_length, + limit_prediction_length=True, + ) + quantiles = model.model.config.chronos_config["quantiles"] + return _forecast_summary(value, task_id, quantiles) + + return Session(invoke, "chronos", compile_evidence=compile_evidence) + + config = transformers.AutoConfig.from_pretrained( + arguments.model, **_processor_kwargs(arguments) + ) + if reference_type == "timesfm": + model = ( + transformers.TimesFmModelForPrediction.from_pretrained( + arguments.model, **_load_kwargs(arguments, torch) + ) + .eval() + .to(device) + ) + length = int(model.config.context_length) + raw = _numeric_values(request, "past_values") + series = torch.tensor(_align(raw, length, 0.0), dtype=dtype, device=device).reshape( + 1, length + ) + observed = _align(_observed_values(request, len(raw)), length, 0.0) + padding = [0 if mask > 0 else 1 for mask in observed] + padding_tensor = torch.tensor(padding, dtype=torch.int32, device=device).reshape(1, length) + frequency = int(request.get("frequency", 0)) + frequency_tensor = torch.tensor([[frequency]], dtype=torch.long, device=device) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + decoder = model.decoder( + past_values=series, + past_values_padding=padding_tensor, + freq=frequency_tensor, + output_attentions=False, + output_hidden_states=False, + ) + output = model._postprocess_output( + decoder.last_hidden_state, (decoder.loc, decoder.scale) + )[:, -1, : model.config.horizon_length, 0] + return _forecast_summary(output, task_id) + + else: + if reference_type not in {"patchtsmixer", "patchtst"}: + raise ValueError(f"unsupported time-series reference_type: {reference_type}") + is_mixer = reference_type == "patchtsmixer" + if is_mixer: + model_class = transformers.PatchTSMixerForPrediction + output_name = "prediction_outputs" + else: + task = _patchtst_task(config) + class_name, output_name = { + "classification": ( + "PatchTSTForClassification", + "prediction_logits", + ), + "forecast": ("PatchTSTForPrediction", "prediction_outputs"), + "regression": ("PatchTSTForRegression", "regression_outputs"), + }[task] + model_class = getattr(transformers, class_name) + if not hasattr(model_class, "all_tied_weights_keys"): + model_class.all_tied_weights_keys = {} + model = ( + model_class.from_pretrained(arguments.model, **_load_kwargs(arguments, torch)) + .eval() + .to(device) + ) + length = int(config.context_length) + channels = int(config.num_input_channels) + raw = _numeric_values(request, "past_values") + values = torch.tensor( + _align(raw, length * channels, 0.0), dtype=dtype, device=device + ).reshape(1, length, channels) + aligned_mask = _align(_observed_values(request, len(raw)), length * channels, 0.0) + observed = torch.tensor( + aligned_mask, + dtype=dtype, + device=device, + ).reshape(1, length, channels) + + def invoke() -> Mapping[str, Any]: + with ( + torch.inference_mode(), + torch.autocast( + device_type="cuda", + dtype=dtype, + enabled=dtype != torch.float32, + ), + ): + if is_mixer: + outputs = model( + past_values=values * observed, + observed_mask=observed, + return_loss=False, + return_dict=True, + ) + output = outputs.prediction_outputs + else: + outputs = model( + past_values=values, + past_observed_mask=observed.gt(0.5), + return_dict=True, + ) + output = getattr(outputs, output_name) + if task_id == "series_to_regression_distribution": + return _regression_summary( + output, config.distribution_output, tuple(model.distribution_output.args_dim) + ) + if task_id == "series_to_regression_values": + return _regression_values_summary(output) + if isinstance(output, (tuple, list)): + output = torch.stack(list(output), dim=-1) + return _forecast_summary(output, task_id) + + return Session(invoke, "transformers") + + +def _load_vision( + arguments: argparse.Namespace, + request: Mapping[str, Any], + options: Mapping[str, Any], +) -> Session: + import torch + from PIL import Image + import transformers + + device = torch.device("cuda") + image = Image.open(_asset_path(arguments, request, "image_path")).convert("RGB") + width, height = image.size + kwargs = _load_kwargs(arguments, torch) + processor_kwargs = _processor_kwargs(arguments) + vision_task = str(options.get("vision_task", "")) + + if arguments.adapter == "timm-classification": + import timm + from timm.data import create_transform, resolve_model_data_config + + model_id = arguments.model + if arguments.revision: + model_id += "@" + arguments.revision + model = timm.create_model(f"hf-hub:{model_id}", pretrained=True) + dtype = _torch_dtype(torch, arguments.precision) + model = model.eval().to(device=device, dtype=dtype) + transform = create_transform(**resolve_model_data_config(model), is_training=False) + inputs = transform(image).unsqueeze(0).to(device=device, dtype=dtype) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + logits = model(inputs) + return {"top_class": int(logits.argmax(dim=-1)[0]), **_tensor_summary(logits)} + + elif vision_task == "object-detection": + processor = transformers.AutoImageProcessor.from_pretrained( + arguments.model, **processor_kwargs + ) + model = ( + transformers.AutoModelForObjectDetection.from_pretrained(arguments.model, **kwargs) + .eval() + .to(device) + ) + inputs = _to_device( + processor(images=image, return_tensors="pt"), + device, + next(model.parameters()).dtype, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + target_sizes = torch.tensor([[height, width]], device=device) + results = processor.post_process_object_detection( + outputs, threshold=0.5, target_sizes=target_sizes + )[0] + return { + "detected_images": 1, + "detections": int(results["scores"].shape[0]), + "image_height": height, + "image_width": width, + "boxes": results["boxes"].float().cpu().reshape(-1).tolist(), + "scores": results["scores"].float().cpu().tolist(), + "class_ids": results["labels"].cpu().tolist(), + "shape": [int(results["scores"].shape[0]), 4], + "coordinates": "xyxy", + "units": "pixels", + } + + elif vision_task == "image-features": + processor = transformers.AutoImageProcessor.from_pretrained( + arguments.model, **processor_kwargs + ) + model = transformers.AutoModel.from_pretrained(arguments.model, **kwargs).eval().to(device) + inputs = _to_device( + processor(images=image, return_tensors="pt"), + device, + next(model.parameters()).dtype, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + return { + "last_hidden_state_shape": _tensor_summary(outputs.last_hidden_state)["shape"], + "pooler_output_shape": _tensor_summary(outputs.pooler_output)["shape"], + } + + elif vision_task == "semantic-segmentation": + processor = transformers.AutoImageProcessor.from_pretrained( + arguments.model, **processor_kwargs + ) + model = ( + transformers.AutoModelForSemanticSegmentation.from_pretrained(arguments.model, **kwargs) + .eval() + .to(device) + ) + inputs = _to_device( + processor(images=image, return_tensors="pt"), + device, + next(model.parameters()).dtype, + ) + + def invoke() -> Mapping[str, Any]: + with torch.inference_mode(): + outputs = model(**inputs) + mask = processor.post_process_semantic_segmentation( + outputs, target_sizes=[(height, width)] + )[0] + return { + "num_masks": 1, + "height": int(mask.shape[0]), + "width": int(mask.shape[1]), + } + + else: + raise ValueError(f"unsupported Transformers vision_task: {vision_task}") + + framework = "timm" if arguments.adapter == "timm-classification" else "transformers" + return Session(invoke, framework) + + + +LOADERS: dict[ + str, Callable[[argparse.Namespace, Mapping[str, Any], Mapping[str, Any]], Session] +] = { + "hf-diffusers": _load_diffusers, + "hf-diffusers-modular": _load_modular_diffusers, + "hf-transformers-asr": _load_asr, + "hf-transformers-embedding": _load_embedding, + "hf-transformers-reranking": _load_reranking, + "hf-transformers-tts": _load_tts, + "hf-transformers-vision": _load_vision, + "hf-transformers-vlm": _load_vlm, + "pytorch-timeseries": _load_timeseries, + "timm-classification": _load_vision, +} + + +def _synchronize() -> None: + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.synchronize() + except ImportError: + return + + +def _compile_forward(model: Any) -> dict[str, Any]: + import torch + from torch._dynamo.backends.registry import lookup_backend + + evidence = {"compiled_graph_count": 0} + inductor = lookup_backend("inductor") + + def compile_graph(graph: Any, inputs: Any, **options: Any) -> Any: + compiled = inductor(graph, inputs, **options) + evidence["compiled_graph_count"] += 1 + return compiled + + model.forward = torch.compile( + model.forward, + backend=compile_graph, + fullgraph=False, + dynamic=False, + ) + evidence.update( + { + "api": "torch.compile", + "target": "model.forward", + "backend": "inductor", + "mode": "default", + "fullgraph": False, + "dynamic": False, + "applied": True, + } + ) + return evidence + + +def _measure(session: Session, warmup: int, iterations: int) -> tuple[list[float], dict[str, Any]]: + output: Any = {} + untimed_iterations = max(warmup, int(session.compile_evidence is not None)) + for _ in range(untimed_iterations): + output = session.invoke() + _synchronize() + compiled_graphs = None + if session.compile_evidence is not None: + compiled_graphs = int(session.compile_evidence["compiled_graph_count"]) + if compiled_graphs < 1: + raise RuntimeError("warmup did not execute a compiled graph") + samples = [] + for _ in range(iterations): + _synchronize() + started = time.perf_counter() + output = session.invoke() + _synchronize() + samples.append((time.perf_counter() - started) * 1000.0) + if ( + compiled_graphs is not None + and int(session.compile_evidence["compiled_graph_count"]) != compiled_graphs + ): + raise RuntimeError("model compilation occurred inside timed samples") + summary = session.summarize(output) if session.summarize is not None else output + return samples, dict(summary) + + +def _environment() -> dict[str, Any]: + value: dict[str, Any] = { + "platform": platform.platform(), + "python": platform.python_version(), + "python_executable": sys.executable, + } + try: + import torch + + value["torch"] = torch.__version__ + if torch.cuda.is_available(): + value["gpu"] = torch.cuda.get_device_name(0) + value["cuda"] = torch.version.cuda + except ImportError: + pass + for module_name in ("transformers", "diffusers", "nemo", "chronos", "moshi"): + try: + module = __import__(module_name) + except ImportError: + continue + value[module_name] = str(getattr(module, "__version__", "unknown")) + return value + + +def run(arguments: argparse.Namespace) -> int: + if arguments.warmup < 0 or arguments.iterations <= 0: + raise ValueError("warmup must be non-negative and iterations must be positive") + options = _json_object(arguments.adapter_options_json, "--adapter-options-json") + expected_mode = "pytorch-eager" if arguments.adapter in PYTORCH_ADAPTERS else "hf-eager" + supported_modes = {expected_mode} + if ( + arguments.adapter == "pytorch-timeseries" + and options.get("reference_type") == "chronos-bolt" + ): + supported_modes.add("torch-compile") + if arguments.adapter == "hf-transformers-embedding": + supported_modes.add("torch-compile") + if arguments.mode not in supported_modes: + raise ValueError(f"adapter {arguments.adapter} requires one of {sorted(supported_modes)}") + request = flatten_config(_json_object(arguments.request_json, "--request-json")) + configured_timing = _json_object(arguments.timing_contract_json, "--timing-contract-json") + fields = ("timing_scope", "input_preparation_included", "asset_loading_included") + expected_timing = None + if configured_timing: + if set(configured_timing) != set(fields): + raise ValueError( + "--timing-contract-json must declare exactly the three reference timing fields" + ) + declared = timing_contract(runner="task-reference", declared=configured_timing) + expected_timing = {name: declared[name] for name in fields} + load_started = time.perf_counter() + load_seconds: float | None = None + compile_evidence: dict[str, Any] | None = None + session = LOADERS[arguments.adapter](arguments, request, options) + compile_evidence = session.compile_evidence + load_seconds = time.perf_counter() - load_started + framework = session.framework + timing_scope = session.timing_scope + input_included = session.input_preparation_included + asset_included = session.asset_loading_included + actual_timing = { + "timing_scope": timing_scope, + "input_preparation_included": input_included, + "asset_loading_included": asset_included, + } + timing_contract(runner="task-reference", declared=actual_timing) + if expected_timing is not None and actual_timing != expected_timing: + raise RuntimeError( + f"{arguments.adapter} reference implementation timing drifted: " + f"actual={actual_timing}, declared={expected_timing}" + ) + samples, output_summary = _measure(session, arguments.warmup, arguments.iterations) + if "_media" in output_summary: + _write_media_artifacts(output_summary, arguments.output) + disparity = output_summary.pop("_disparity_f32", None) + if disparity is not None: + artifact_path = arguments.output.with_suffix(".disparity.f32").resolve() + artifact_path.parent.mkdir(parents=True, exist_ok=True) + disparity.tofile(artifact_path) + output_summary["disparity_artifact"] = str(artifact_path) + audio = output_summary.pop("_audio_f32", None) + if audio is not None: + import soundfile as sf + + artifact_path = arguments.output.with_suffix(".audio.wav").resolve() + artifact_path.parent.mkdir(parents=True, exist_ok=True) + sf.write(artifact_path, audio, output_summary["sample_rate"], subtype="FLOAT") + output_summary["audio_artifact"] = str(artifact_path) + result = { + "schema_version": "trtmc.perf-baseline/v1", + "status": "completed", + "backend": arguments.adapter, + "adapter": arguments.adapter, + "framework": framework, + "mode": arguments.mode, + "precision": arguments.precision, + "padding": arguments.padding, + "experts_implementation": None, + "compile_scope": "model.forward" if arguments.mode == "torch-compile" else None, + "compile_evidence": compile_evidence, + "timing_scope": timing_scope, + "input_preparation_included": input_included, + "asset_loading_included": asset_included, + "model_load_included": False, + "model_load_seconds": load_seconds, + "model": arguments.model, + "case_name": arguments.case_name, + "measurement": { + "warmup": arguments.warmup, + "iterations": arguments.iterations, + }, + "measurement_policy": { + "timing_scope": timing_scope, + "input_preparation_included": input_included, + "asset_loading_included": asset_included, + "model_load_excluded": True, + "compile_excluded": True, + "warmup_excluded": True, + "output_materialization_included": True, + }, + "samples_ms": samples, + "metrics": { + "sample_count": len(samples), + "latency_ms": { + "p50": statistics.median(samples), + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + }, + }, + "output_summary": output_summary, + "environment": _environment(), + "finished_at": datetime.now(timezone.utc).isoformat(), + } + if compile_evidence is not None: + compile_evidence["warmup_completed"] = True + compile_evidence["timed_callable_uses_compiled_target"] = True + if not all(math.isfinite(float(value)) and float(value) > 0.0 for value in samples): + raise RuntimeError("reference produced an invalid timing sample") + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + try: + arguments = build_parser().parse_args(argv) + if arguments.local_files_only: + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + return run(arguments) + except (KeyError, OSError, RuntimeError, TypeError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/benchmark/performance/baselines/hf_transformers.py b/qualification_tests/benchmark_qualification/performance/references/hf_transformers.py similarity index 88% rename from apps/benchmark/performance/baselines/hf_transformers.py rename to qualification_tests/benchmark_qualification/performance/references/hf_transformers.py index 47d37f097b..f81030f405 100644 --- a/apps/benchmark/performance/baselines/hf_transformers.py +++ b/qualification_tests/benchmark_qualification/performance/references/hf_transformers.py @@ -179,13 +179,20 @@ def _batch_prompt(request: Mapping[str, Any]) -> list[str]: return [prompt] * batch_size -def _translation_controls(tokenizer: Any, request: Mapping[str, Any]) -> dict[str, int]: +def _translation_controls( + tokenizer: Any, request: Mapping[str, Any] +) -> tuple[dict[str, int], int | None]: """Use tokenizer language APIs; fixed-pair models validate their declared pair.""" source = request.get("source_language") target = request.get("target_language") + source_placement = request.get("source_language_placement") for name, value in (("source_language", source), ("target_language", target)): if value is not None and (not isinstance(value, str) or not value): raise ValueError(f"request.{name} must be a nonempty language identifier") + if source_placement is not None and source_placement != "replace-final-unk": + raise ValueError( + "request.source_language_placement must be replace-final-unk when set" + ) def token_id(language: str) -> int: lookup = getattr(tokenizer, "get_lang_id", None) @@ -211,6 +218,11 @@ def explicit_id(name: str) -> int | None: source_id = explicit_id("source_language_token_id") target_id = explicit_id("forced_bos_token_id") + if source_placement is not None and source_id is None: + raise ValueError( + "request.source_language_placement requires source_language_token_id" + ) + manual_source_id = None if source_id is not None: if source is not None and token_id(source) != source_id: raise ValueError("source language disagrees with source_language_token_id") @@ -219,7 +231,16 @@ def explicit_id(name: str) -> int | None: if hasattr(tokenizer, "src_lang"): token_id(source) tokenizer.src_lang = source - elif source != getattr(tokenizer, "source_lang", None): + elif source == getattr(tokenizer, "source_lang", None): + pass + elif source_id is not None and token_id(source) == source_id: + if source_placement != "replace-final-unk": + raise ValueError( + "generic translation tokenizer requires explicit " + "source_language_placement=replace-final-unk" + ) + manual_source_id = source_id + else: raise ValueError("reference tokenizer does not support the requested source language") if target is not None: if hasattr(tokenizer, "src_lang"): @@ -227,9 +248,39 @@ def explicit_id(name: str) -> int | None: if target_id is not None and target_id != resolved: raise ValueError("target language disagrees with forced_bos_token_id") target_id = resolved - elif target != getattr(tokenizer, "target_lang", None): + elif target == getattr(tokenizer, "target_lang", None): + pass + elif target_id is not None and token_id(target) == target_id: + pass + else: raise ValueError("reference tokenizer does not support the requested target language") - return {} if target_id is None else {"forced_bos_token_id": target_id} + controls = {} if target_id is None else {"forced_bos_token_id": target_id} + return controls, manual_source_id + + +def _apply_source_language( + encoded: Mapping[str, Any], source_token_id: int | None, tokenizer: Any +) -> None: + if source_token_id is None: + return + input_ids = encoded.get("input_ids") + if input_ids is None or getattr(input_ids, "ndim", None) != 2: + raise ValueError("tokenized translation input must contain rank-2 input_ids") + attention_mask = encoded.get("attention_mask") + for row in range(int(input_ids.shape[0])): + if attention_mask is None: + index = int(input_ids.shape[1]) - 1 + else: + positions = attention_mask[row].nonzero(as_tuple=False) + if int(positions.numel()) == 0: + raise ValueError("tokenized translation input cannot be empty") + index = int(positions[-1].item()) + current = int(input_ids[row, index].item()) + if current == source_token_id: + continue + if current != getattr(tokenizer, "unk_token_id", None): + raise ValueError("generic translation tokenizer did not emit a language placeholder") + input_ids[row, index] = source_token_id def _encoder_call( @@ -321,10 +372,11 @@ def encode_prompt() -> Any: generation: dict[str, Any] = { "do_sample": float(request.get("temperature", 0.0)) > 0.0, "max_new_tokens": max_new_tokens, + "num_beams": 1, "use_cache": True, "pad_token_id": tokenizer.pad_token_id, } - translation = _translation_controls(tokenizer, request) + translation, source_language_token_id = _translation_controls(tokenizer, request) if generation_method == "ar-generate" and translation: raise ValueError("ar-generate reference does not accept translation controls") generation.update(translation) @@ -345,6 +397,7 @@ def encode_prompt() -> Any: def invoke() -> dict[str, Any]: encoded = encode_prompt() + _apply_source_language(encoded, source_language_token_id, tokenizer) gpu_inputs = {name: value.to("cuda") for name, value in encoded.items()} with ( torch.inference_mode(), diff --git a/apps/benchmark/performance/baselines/timing_contracts.py b/qualification_tests/benchmark_qualification/performance/references/timing_contracts.py similarity index 100% rename from apps/benchmark/performance/baselines/timing_contracts.py rename to qualification_tests/benchmark_qualification/performance/references/timing_contracts.py diff --git a/qualification_tests/benchmark_qualification/performance/reporting.py b/qualification_tests/benchmark_qualification/performance/reporting.py new file mode 100644 index 0000000000..144ebe7cbf --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/reporting.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-independent JSON and HTML reporting for Performance qualification.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import html +import json +import math +from pathlib import Path +from typing import Any, Mapping + +from .types import PerfMatrixError + + +RESULT_SCHEMA = "trtmc.perf-matrix/v2" +REPORT_SCHEMA = "trtmc.perf-report/v2" + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_report( + run_directory: Path, + results: Mapping[str, Any], + preparation: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + source_schema = results.get("schema_version", RESULT_SCHEMA) + row_key = "cases" if source_schema == "trtmc.perf-matrix/v1" else "rows" + rows = [dict(row) for row in results.get(row_key, []) if isinstance(row, Mapping)] + selected_ids = results.get("selected_entry_ids") + if not isinstance(selected_ids, list) or not all( + isinstance(value, str) for value in selected_ids + ): + raise PerfMatrixError("matrix results selected_entry_ids must be strings") + completed_ids = {str(row.get("id")) for row in rows} + counts = { + status: sum(row.get("status") == status for row in rows) + for status in ("green", "yellow", "red", "contract-mismatch", "white") + } + report = { + "schema_version": REPORT_SCHEMA, + "source_schema_version": source_schema, + "generated_at": _now(), + "status": results.get("status", "unknown"), + "suite": results.get("suite"), + "environment": results.get("environment"), + "summary": { + "selected": len(selected_ids), + "pending": sum(entry_id not in completed_ids for entry_id in selected_ids), + "comparable": counts["green"] + counts["yellow"] + counts["red"], + **counts, + }, + "rows": rows, + } + if preparation is not None: + report["preparation"] = dict(preparation) + _write_json(run_directory / "report.json", report) + (run_directory / "report.html").write_text(_report_html(report), encoding="utf-8") + return report + + +def _measurement_html(value: Any) -> str: + if not isinstance(value, Mapping): + return "Not recorded" + metrics = value.get("metrics", {}) + latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} + samples = value.get("samples_ms", []) + measured = ( + sorted( + float(sample) + for sample in samples + if isinstance(sample, (int, float)) + and not isinstance(sample, bool) + and math.isfinite(sample) + and sample > 0 + ) + if isinstance(samples, list) + else [] + ) + parts = [] + for name, percentile in (("p50", 0.5), ("p95", 0.95)): + number = latency.get(name) if isinstance(latency, Mapping) else None + if number is None and measured: + position = (len(measured) - 1) * percentile + lower, upper = math.floor(position), math.ceil(position) + number = measured[lower] + (measured[upper] - measured[lower]) * (position - lower) + if ( + isinstance(number, (int, float)) + and not isinstance(number, bool) + and math.isfinite(number) + ): + parts.append(f"{name}: {number:,.3f} ms") + policy = value.get("measurement_policy", {}) + scope = value.get("timing_scope") or ( + policy.get("timing_scope") if isinstance(policy, Mapping) else None + ) + parts.append("Scope: " + html.escape(str(scope or "not recorded"))) + if isinstance(metrics, Mapping): + parts.extend( + f"{html.escape(name)}: {number:,.3f}" + for name, number in metrics.items() + if name.endswith("_per_s") + and isinstance(number, (int, float)) + and not isinstance(number, bool) + and math.isfinite(number) + ) + return "
".join(parts) + + +def _report_html(report: Mapping[str, Any]) -> str: + rows = [] + for row in report.get("rows", []): + comparison = row.get("comparison", {}) if isinstance(row, Mapping) else {} + measurement_stability = ( + row.get("measurement_stability", {}) if isinstance(row, Mapping) else {} + ) + candidate = _measurement_html( + row.get("candidate") + or {"metrics": {"latency_ms": {"p50": comparison.get("candidate_p50_ms")}}} + ) + reference = _measurement_html( + row.get("reference", row.get("baseline")) + or {"metrics": {"latency_ms": {"p50": comparison.get("reference_p50_ms")}}} + ) + reason = comparison.get("reason", row.get("error", "")) + stability = ( + measurement_stability.get("status", "") + if isinstance(measurement_stability, Mapping) + else "" + ) + evidence = { + key: row[key] + for key in ( + "commands", + "comparison", + "measurement_stability", + "resolved_settings", + "baseline_contract", + ) + if key in row + } + detail = html.escape(json.dumps(evidence, indent=2, sort_keys=True)) + link = "" + artifact = row.get("artifact_dir") + if ( + isinstance(artifact, str) + and artifact + and not Path(artifact).is_absolute() + and ".." not in Path(artifact).parts + and not any(value in artifact for value in (":", "\\")) + ): + from urllib.parse import quote + + link = f'Case artifacts' + rows.append( + "" + f"{html.escape(str(row.get('id', '')))}" + f"{html.escape(str(row.get('model', '')))}" + f"{html.escape(str(row.get('operation', '')))}" + f"{html.escape(str(row.get('status', '')))}" + f"{html.escape(str(stability))}" + f"{candidate}" + f"{reference}" + f"{html.escape(str(reason))}" + f"
Evidence / commands{link}
{detail}
" + "" + ) + return f""" +TRTMC performance + +

TRTMC performance matrix

+

Source data: {html.escape(str(report.get("source_schema_version", RESULT_SCHEMA)))}. Original timing scopes and recorded outcomes are preserved.

+

Status: {html.escape(str(report.get("status", "unknown")))}; +selected: {html.escape(str(report.get("summary", {}).get("selected", 0)))}; +pending: {html.escape(str(report.get("summary", {}).get("pending", 0)))}

+ + + +{"".join(rows)}
EntryModelOperationStatusStabilityCandidate measurementsReference measurementsReasonEvidence
+

Green is faster by more than the margin, yellow is within the margin, red is +slower by more than the margin, and white is not comparable.

+

Output comparison follows the suite's declared contract; matching dimensions or token counts alone does not establish model correctness. Use family correctness tests for acceptance.

+

Original results · Report data

+ + +""" diff --git a/qualification_tests/benchmark_qualification/performance/runner.py b/qualification_tests/benchmark_qualification/performance/runner.py new file mode 100644 index 0000000000..277a657813 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/runner.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Execute one internal Performance Qualification Target.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +from typing import Any, Mapping + +from . import matrix + + +def run_case( + spec: Mapping[str, Any], + environment: matrix.Environment, + output: Path, + *, + no_build: bool, + verbose: bool, +) -> dict[str, Any]: + """Run one case through the same execution contract as a campaign matrix.""" + + resolved = matrix.preflight([dict(spec)], environment, require_runtime=True) + if len(resolved) != 1: + raise matrix.PerfMatrixError("single-case qualification resolved an unexpected case count") + output.mkdir(parents=True, exist_ok=True) + entry = resolved[0] + try: + result = matrix.execute_entry( + entry, + environment, + output, + no_build=no_build, + verbose=verbose, + attempt=1, + ) + except (OSError, matrix.PerfMatrixError, subprocess.SubprocessError) as error: + result = { + "id": str(entry.spec["id"]), + "model": entry.model.name, + "family": entry.model.family, + "operation": entry.spec["operation"], + "testcase": entry.case.testcase_name, + "status": "white", + "attempts": 1, + "error": str(error), + } + receipt = { + "schema_version": matrix.RESULT_SCHEMA, + "status": ( + "completed" if result.get("status") in matrix.TERMINAL_COMPARISONS else "failed" + ), + "suite": str(spec["id"]), + "environment": environment.name, + "selected_entry_ids": [str(spec["id"])], + "rows": [result], + } + (output / "results.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + matrix.write_report(output, receipt) + return result diff --git a/qualification_tests/benchmark_qualification/performance/stability.py b/qualification_tests/benchmark_qualification/performance/stability.py new file mode 100644 index 0000000000..910ed18a45 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/stability.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-independent timing stability policy.""" + +from __future__ import annotations + +import math +import statistics +from typing import Any, Mapping, Sequence + +from .types import PerfMatrixError + + +SAMPLE_COUNT = 10 +MAX_HALF_MEDIAN_CHANGE_PERCENT = 5.0 +MEDIAN_BAND_PERCENT = 5.0 +MINIMUM_SAMPLES_IN_BAND = 8 + + +def p50(value: Mapping[str, Any]) -> float: + metrics = value.get("metrics", {}) + latency = metrics.get("latency_ms", {}) if isinstance(metrics, Mapping) else {} + value_p50 = latency.get("p50") if isinstance(latency, Mapping) else None + if ( + isinstance(value_p50, bool) + or not isinstance(value_p50, (int, float)) + or not math.isfinite(float(value_p50)) + ): + raise PerfMatrixError("measurement has no finite latency p50") + return float(value_p50) + + +def timing_stability(values: Sequence[Any]) -> dict[str, Any]: + if len(values) != SAMPLE_COUNT: + return { + "status": "not_evaluated", + "sample_count": len(values), + "reason": "requires_10_samples", + } + try: + samples = [float(value) for value in values] + except (TypeError, ValueError): + return { + "status": "not_evaluated", + "sample_count": len(values), + "reason": "invalid_samples", + } + if not all(math.isfinite(value) and value > 0.0 for value in samples): + return { + "status": "not_evaluated", + "sample_count": len(samples), + "reason": "invalid_samples", + } + middle = len(samples) // 2 + median_ms = statistics.median(samples) + first_half_median_ms = statistics.median(samples[:middle]) + second_half_median_ms = statistics.median(samples[middle:]) + half_change_percent = ( + abs(second_half_median_ms - first_half_median_ms) / first_half_median_ms * 100.0 + ) + samples_within_band = sum( + abs(sample - median_ms) / median_ms * 100.0 <= MEDIAN_BAND_PERCENT for sample in samples + ) + stable = ( + half_change_percent <= MAX_HALF_MEDIAN_CHANGE_PERCENT + and samples_within_band >= MINIMUM_SAMPLES_IN_BAND + ) + return { + "status": "stable" if stable else "unstable", + "sample_count": len(samples), + "median_ms": median_ms, + "first_half_median_ms": first_half_median_ms, + "second_half_median_ms": second_half_median_ms, + "half_median_change_percent": half_change_percent, + "samples_within_band": samples_within_band, + } + + +def measurement_stability( + reference: Mapping[str, Any], candidate: Mapping[str, Any] +) -> dict[str, Any]: + reference_result = timing_stability(reference.get("samples_ms", [])) + candidate_result = timing_stability(candidate.get("samples_ms", [])) + statuses = {reference_result["status"], candidate_result["status"]} + if statuses == {"stable"}: + status = "stable" + elif "unstable" in statuses: + status = "unstable" + else: + status = "not_evaluated" + return { + "status": status, + "policy": { + "required_samples": SAMPLE_COUNT, + "max_half_median_change_percent": MAX_HALF_MEDIAN_CHANGE_PERCENT, + "median_band_percent": MEDIAN_BAND_PERCENT, + "minimum_samples_within_band": MINIMUM_SAMPLES_IN_BAND, + }, + "reference": reference_result, + "candidate": candidate_result, + } diff --git a/qualification_tests/benchmark_qualification/performance/tests/__init__.py b/qualification_tests/benchmark_qualification/performance/tests/__init__.py new file mode 100644 index 0000000000..0eb8c14ecc --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/tests/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Performance Qualification contract tests.""" diff --git a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py b/qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py similarity index 82% rename from apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py rename to qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py index 8655d97073..06a1d0ee46 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_diffusers_semantic_reference.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py @@ -14,7 +14,7 @@ import numpy as np import pytest -from apps.benchmark.performance.baselines import task_reference +from qualification_tests.benchmark_qualification.performance.references import generic_reference @pytest.fixture @@ -164,7 +164,7 @@ def arguments(class_name="FluxPipeline", family="flux", task="text_to_image", ** def test_prompt_and_seed_order_reaches_every_pipeline_invocation(recording_diffusers, payload, expected_prompt, expected_seeds): recorded = recording_diffusers source = deepcopy(payload) - session = task_reference._load_diffusers(recorded.arguments(), task_reference.flatten_config(payload), {}) + session = generic_reference._load_diffusers(recorded.arguments(), generic_reference.flatten_config(payload), {}) summaries = [session.invoke() for _ in range(3)] assert len(recorded.loads) == len(recorded.constructors) == 1 assert len(recorded.calls) == 3 @@ -172,17 +172,20 @@ def test_prompt_and_seed_order_reaches_every_pipeline_invocation(recording_diffu assert all(call["observed_seeds"] == expected_seeds for call in recorded.calls) assert [generator.resets for generator in recorded.generators] == [[seed] * 4 for seed in expected_seeds] assert all(generator.device == "cuda" for generator in recorded.generators) - assert all(summary == {"media_type": "image", "media_count": len(expected_seeds), - "height": 8, "width": 10, "channels": 3, "finite": True} for summary in summaries) + for summary in summaries: + media = summary.pop("_media") + assert np.asarray(media).shape == (len(expected_seeds), 8, 10, 3) + assert summary == {"media_type": "image", "media_count": len(expected_seeds), + "height": 8, "width": 10, "channels": 3, "finite": True} assert payload == source def test_measurement_resets_each_seed_after_warmups(recording_diffusers): recorded = recording_diffusers - session = task_reference._load_diffusers( + session = generic_reference._load_diffusers( recorded.arguments(), {"prompt": ["second", "first"], "seeds": [2**32 - 1, 0]}, {}, ) - samples, summary = task_reference._measure(session, warmup=2, iterations=3) + samples, summary = generic_reference._measure(session, warmup=2, iterations=3) assert len(samples) == 3 assert len(recorded.calls) == 5 assert all(call["observed_seeds"] == [2**32 - 1, 0] for call in recorded.calls) @@ -209,7 +212,7 @@ def test_measurement_resets_each_seed_after_warmups(recording_diffusers): def test_invalid_batch_requests_do_not_call_pipeline(recording_diffusers, payload, match): recorded = recording_diffusers with pytest.raises(ValueError, match=match): - task_reference._load_diffusers(recorded.arguments(), payload, {}) + generic_reference._load_diffusers(recorded.arguments(), payload, {}) assert recorded.calls == [] @@ -217,12 +220,20 @@ def test_invalid_batch_requests_do_not_call_pipeline(recording_diffusers, payloa @pytest.mark.parametrize("class_name", ["QwenImageEditPipeline", "QwenImageEditPlusPipeline"]) def test_qwen_edit_checkpoint_selects_edit_class_and_receives_full_request(recording_diffusers, tmp_path, image_key, class_name): recorded = recording_diffusers - arguments = recorded.arguments(class_name, "qwen_image", "images_text_to_image_edit") + arguments = recorded.arguments(class_name, "synthetic", "images_text_to_image_edit") image = tmp_path / "condition.ppm" image.write_bytes(b"P6\n1 1\n255\n\xff\x00\x80") request = {"prompt": "Make it blue", image_key: [image.name] if image_key == "image_paths" else image.name, "config": {"seed": 2**32 - 1, "num_steps": 5, "guidance_scale": 3.25, "negative_prompt": " "}} - session = task_reference._load_diffusers(arguments, task_reference.flatten_config(request), {"cpu_offload": True}) + session = generic_reference._load_diffusers( + arguments, + generic_reference.flatten_config(request), + { + "cpu_offload": True, + "always_negative_prompt": True, + "guidance_parameter": "true_cfg_scale", + }, + ) session.invoke() pipeline = recorded.constructors[0] assert pipeline.__class__.__name__ == class_name @@ -253,8 +264,12 @@ def test_qwen_edit_checkpoint_selects_edit_class_and_receives_full_request(recor @pytest.mark.parametrize("negative", [{}, {"negative_prompt": ""}]) def test_qwen_native_guidance_controls_true_cfg_and_preserves_empty_negative_prompt(recording_diffusers, controls, expected, negative): recorded = recording_diffusers - arguments = recorded.arguments("QwenImagePipeline", "qwen_image") - session = task_reference._load_diffusers(arguments, {"prompt": "a", **negative, **controls}, {}) + arguments = recorded.arguments("QwenImagePipeline", "synthetic") + session = generic_reference._load_diffusers( + arguments, + {"prompt": "a", **negative, **controls}, + {"always_negative_prompt": True, "guidance_parameter": "true_cfg_scale"}, + ) session.invoke() assert recorded.calls[0]["true_cfg_scale"] == expected assert recorded.calls[0]["guidance_scale"] is None @@ -263,7 +278,7 @@ def test_qwen_native_guidance_controls_true_cfg_and_preserves_empty_negative_pro def test_non_qwen_guidance_retains_provider_parameter(recording_diffusers): recorded = recording_diffusers - task_reference._load_diffusers(recorded.arguments(), {"prompt": "a", "guidance_scale": 1.5}, {}).invoke() + generic_reference._load_diffusers(recorded.arguments(), {"prompt": "a", "guidance_scale": 1.5}, {}).invoke() assert recorded.calls[0]["guidance_scale"] == 1.5 assert recorded.constructors[0].devices == ["cuda"] @@ -277,32 +292,50 @@ def test_non_qwen_guidance_retains_provider_parameter(recording_diffusers): ]) def test_qwen_edit_image_contract_fails_before_invocation(recording_diffusers, inputs, match): recorded = recording_diffusers - arguments = recorded.arguments("QwenImageEditPlusPipeline", "qwen_image", "images_text_to_image_edit") + arguments = recorded.arguments( + "QwenImageEditPlusPipeline", "synthetic", "images_text_to_image_edit" + ) with pytest.raises(ValueError, match=match): - task_reference._load_diffusers(arguments, {"prompt": "edit", **inputs}, {}) + generic_reference._load_diffusers(arguments, {"prompt": "edit", **inputs}, {}) assert recorded.calls == recorded.images == [] def test_conditioning_image_is_never_silently_filtered(recording_diffusers): recorded = recording_diffusers - arguments = recorded.arguments("QwenImagePipeline", "qwen_image", "images_text_to_image_edit") + arguments = recorded.arguments("QwenImagePipeline", "synthetic", "images_text_to_image_edit") with pytest.raises(ValueError, match="does not accept conditioning image"): - task_reference._load_diffusers(arguments, {"prompt": "edit", "image_paths": ["condition.ppm"]}, {}) + generic_reference._load_diffusers(arguments, {"prompt": "edit", "image_paths": ["condition.ppm"]}, {}) assert recorded.calls == [] -@pytest.mark.parametrize("family,class_name,contract,component_key,subfolder", [ - ("pixart", "PixArtSigmaPipeline", "pixart_fp16_dit_fp32_t5", "text_encoder", "text_encoder"), - ("wan2_2_ti2v", "WanPipeline", "", "vae", "vae"), +@pytest.mark.parametrize("class_name,options,component_key,subfolder", [ + ( + "PixArtSigmaPipeline", + {"component_precision_contract": "pixart_fp16_dit_fp32_t5"}, + "text_encoder", + "text_encoder", + ), + ( + "WanPipeline", + {"vae_class": "AutoencoderKLWan", "vae_precision": "fp32"}, + "vae", + "vae", + ), ]) @pytest.mark.parametrize("local_files_only", [True, False]) def test_checkpoint_owned_loader_preserves_component_precision_and_revision( - recording_diffusers, family, class_name, contract, component_key, subfolder, local_files_only, + recording_diffusers, class_name, options, component_key, subfolder, local_files_only, ): recorded = recording_diffusers - arguments = recorded.arguments(class_name, family, precision="fp16", local_files_only=local_files_only) - options = {"component_precision_contract": contract, "model_revision": "override-revision", "trust_remote_code": True} - session = task_reference._load_diffusers(arguments, {"prompt": "a"}, options) + arguments = recorded.arguments( + class_name, "synthetic", precision="fp16", local_files_only=local_files_only + ) + options = { + **options, + "model_revision": "override-revision", + "trust_remote_code": True, + } + session = generic_reference._load_diffusers(arguments, {"prompt": "a"}, options) session.invoke() source, kwargs, component = recorded.components[0] expected_source = Path(arguments.model) if local_files_only else arguments.model @@ -313,7 +346,7 @@ def test_checkpoint_owned_loader_preserves_component_precision_and_revision( "torch_dtype": "fp16", component_key: component, "local_files_only": local_files_only, "trust_remote_code": True, **revision})] assert recorded.constructors[0].__class__.__name__ == class_name - if contract: + if options.get("component_precision_contract"): transformer = recorded.constructors[0].transformer assert len(transformer.pre_hooks) == len(transformer.hooks) == 1 assert transformer.pre_hooks[0][1] == {"with_kwargs": True} @@ -323,10 +356,10 @@ def test_model_id_override_and_missing_local_snapshot_keep_load_contract(recordi recorded = recording_diffusers arguments = recorded.arguments() override = recorded.arguments("Flux2Pipeline").model - pipeline = task_reference._diffusion_pipeline(arguments, recorded.torch, {"model_id": override}) + pipeline = generic_reference._diffusion_pipeline(arguments, recorded.torch, {"model_id": override}) assert pipeline.__class__.__name__ == "Flux2Pipeline" assert recorded.loads[0][1] == Path(override) - monkeypatch.setattr(task_reference, "_cached_snapshot_path", lambda *_: None) + monkeypatch.setattr(generic_reference, "_cached_snapshot_path", lambda *_: None) with pytest.raises(FileNotFoundError, match="cached Diffusers snapshot is missing"): - task_reference._diffusion_pipeline(arguments, recorded.torch, {}) + generic_reference._diffusion_pipeline(arguments, recorded.torch, {}) assert len(recorded.loads) == 1 diff --git a/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py b/qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py similarity index 96% rename from apps/benchmark/trtmc_benchmark/tests/test_family_performance.py rename to qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py index 4e73e48de7..bbe9017ff4 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_family_performance.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py @@ -15,7 +15,8 @@ import pytest import yaml -import tools.perf_matrix as perf +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance.references import generic_reference REFERENCE_SCRIPT = '''"""Synthetic reference fixture; no model or GPU qualification.""" @@ -78,6 +79,22 @@ def invoke(): ''' +def test_generic_reference_has_no_family_dispatch() -> None: + source = Path(generic_reference.__file__).read_text(encoding="utf-8").casefold() + for family in ( + "magpie_tts", + "canary", + "nemotron_speech_streaming", + "glmasr", + "detr", + "dinov3", + "segformer", + "sam3", + ): + assert family not in source + assert "arguments.family" not in source + + def test_custom_script_default_mode_does_not_claim_torch_compilation(): assert perf._baseline_mode({"script": "tests/reference.py"}) == "reference" assert perf._baseline_mode({"script": "tests/reference.py", "mode": "pytorch-eager"}) == "pytorch-eager" @@ -125,7 +142,7 @@ def repository(tmp_path, monkeypatch): (folder / "manifests" / (owner + ".json")).write_text(json.dumps(manifest)) script = families / "sample/tests/performance_reference.py" script.write_text(REFERENCE_SCRIPT, encoding="utf-8") - canonical = root / "apps/benchmark/performance/release.yaml" + canonical = root / "qualification_tests/benchmark_qualification/performance/config/release.yaml" _write_suite(canonical, [_entry("other", "other-model")]) owned_suite = families / "sample/tests/performance.yaml" _write_suite(owned_suite, [_entry("sample", "sample-model", script=True)]) @@ -761,6 +778,27 @@ def test_existing_adapter_command_keeps_existing_arguments(repository): assert command[command.index("--manifest") + 1] == str(entry.model.manifest_path) +def test_hf_reference_can_use_an_independent_pinned_model(repository): + entry = _resolved(repository) + baseline = { + key: value for key, value in entry.spec["baseline"].items() if key != "script" + } + baseline.update( + runner="hf-transformers", + model="fixture/reference", + revision="b" * 40, + ) + entry = replace(entry, spec={**entry.spec, "baseline": baseline}) + + environment = SimpleNamespace( + **vars(repository.environment), reference_python=Path(sys.executable) + ) + command = perf.baseline_command(entry, environment, repository.root / "out.json") + + assert command[command.index("--model") + 1] == "fixture/reference" + assert command[command.index("--revision") + 1] == "b" * 40 + + @pytest.mark.parametrize("failure", [None, "process", "identity"]) def test_execute_entry_validates_actual_script_process(repository, tmp_path, monkeypatch, failure): entry = _resolved(repository) @@ -789,9 +827,9 @@ def test_execute_entry_validates_actual_script_process(repository, tmp_path, mon if failure: expected = "reference command failed" if failure == "process" else "case_name" with pytest.raises(perf.PerfMatrixError, match=expected): - perf._execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) + perf.execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) else: - row = perf._execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) + row = perf.execute_entry(entry, environment, tmp_path / "run", no_build=True, verbose=False, attempt=1) assert row["reference"]["output_summary"]["loaded_models"] == 1 assert row["reference"]["output_summary"]["invocations"] == 5 # The existing ten-sample performance gate is intentionally unchanged. diff --git a/apps/benchmark/trtmc_benchmark/tests/test_offline_speech_contract.py b/qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py similarity index 99% rename from apps/benchmark/trtmc_benchmark/tests/test_offline_speech_contract.py rename to qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py index a0afa46ad5..8fa76b1f42 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_offline_speech_contract.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py @@ -9,7 +9,7 @@ import pytest -import tools.perf_matrix as perf +from qualification_tests.benchmark_qualification.performance import matrix as perf def _entry(task="offline_speech_dialogue", *, primary="duplex_speech_dialogue", explicit=False): diff --git a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py b/qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py similarity index 53% rename from apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py rename to qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py index 17888977e7..892b52133c 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_perf_matrix.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py @@ -3,6 +3,7 @@ from __future__ import annotations +import ast import sys from contextlib import nullcontext from copy import deepcopy @@ -16,17 +17,43 @@ import pytest import yaml -import tools.perf_matrix as perf -from apps.benchmark.performance.baselines import ( +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance.references import ( hf_transformers, - lance_reference, - sana_wm_reference, - task_reference, + generic_reference, ) +from apps.benchmark.trtmc_benchmark.types import ModelDescriptor REPO = Path(__file__).resolve().parents[4] -SUITE = REPO / "apps/benchmark/performance/release.yaml" +SUITE = REPO / "qualification_tests/benchmark_qualification/performance/config/release.yaml" + + +def test_shared_reference_runner_never_uses_family_identity_to_select_behavior() -> None: + tree = ast.parse(Path(generic_reference.__file__).read_text(encoding="utf-8")) + parents: dict[ast.AST, ast.AST] = { + child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent) + } + family_reads = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and node.attr == "family" + and isinstance(node.value, ast.Name) + and node.value.id == "arguments" + ] + controls = [ + node.test + for node in ast.walk(tree) + if isinstance(node, (ast.If, ast.IfExp)) + ] + [node.subject for node in ast.walk(tree) if isinstance(node, ast.Match)] + violations = [ + node.lineno + for node in family_reads + if any(node in ast.walk(control) for control in controls) + or (isinstance(parents.get(node), ast.Call) and node in parents[node].args) + ] + assert violations == [] def test_reference_config_preserves_explicit_values_without_defaults() -> None: @@ -100,15 +127,15 @@ def __call__(self, prompt, height=None, width=None): pil = ModuleType("PIL") pil.Image = SimpleNamespace() monkeypatch.setitem(sys.modules, "PIL", pil) - monkeypatch.setattr(task_reference, "_diffusion_pipeline", lambda *_: Pipeline() if accepts_frames else ImagePipeline()) + monkeypatch.setattr(generic_reference, "_diffusion_pipeline", lambda *_: Pipeline() if accepts_frames else ImagePipeline()) original = perf.ManifestCatalog(REPO / "families").resolve(selector) model = replace(original, testcases=({**original.testcases[0], **controls},)) case = perf.resolve_case(model, tmp_path / "model.bundle", selected_task=task) candidate_request = json.loads(json.dumps(case.request)) - request = task_reference.flatten_config(case.request) + request = generic_reference.flatten_config(case.request) before = dict(request) arguments = SimpleNamespace(manifest=model.manifest_path, family=model.family, precision=model.precision) - task_reference._load_diffusers(arguments, request, {}).invoke() + generic_reference._load_diffusers(arguments, request, {}).invoke() wanted_height, wanted_width, wanted_frames = expected if expected is not None else (height, width, frames) assert captured["height"] == wanted_height assert captured["width"] == wanted_width @@ -132,15 +159,15 @@ def convert_ids_to_tokens(self, token): return {10: "eng_Latn", 11: "fra_Latn"}[token] tokenizer = Tokenizer() - assert hf_transformers._translation_controls(tokenizer, {}) == {} + assert hf_transformers._translation_controls(tokenizer, {}) == ({}, None) assert tokenizer.src_lang == "default" assert hf_transformers._translation_controls( tokenizer, {"source_language": "eng_Latn", "target_language": "fra_Latn"} - ) == {"forced_bos_token_id": 11} + ) == ({"forced_bos_token_id": 11}, None) assert tokenizer.src_lang == "eng_Latn" assert hf_transformers._translation_controls( tokenizer, {"source_language_token_id": 10, "forced_bos_token_id": 0} - ) == {"forced_bos_token_id": 0} + ) == ({"forced_bos_token_id": 0}, None) with pytest.raises(ValueError, match="disagrees"): hf_transformers._translation_controls( tokenizer, {"target_language": "fra_Latn", "forced_bos_token_id": 10} @@ -152,7 +179,7 @@ def convert_ids_to_tokens(self, token): hf_transformers._translation_controls( fixed, {"source_language": "en", "target_language": "ru"} ) - == {} + == ({}, None) ) with pytest.raises(ValueError, match="target language"): hf_transformers._translation_controls(fixed, {"target_language": "de"}) @@ -200,6 +227,7 @@ def decode(self, tokens, **_kwargs): class Model: config = SimpleNamespace(decoder_start_token_id=0, eos_token_id=3) + generation_config = SimpleNamespace(num_beams=4) def generate(self, **kwargs): captured.update(kwargs) @@ -228,10 +256,73 @@ def generate(self, **kwargs): assert captured["input_ids"].tolist() == [[10, 17]] assert captured["forced_bos_token_id"] == 11 assert captured["max_new_tokens"] == 5 and captured["do_sample"] is False + assert captured["num_beams"] == 1 assert captured["repetition_penalty"] == 1.1 assert output["token_ids"] == [8] +def test_reference_only_source_language_placement_reaches_hf_runner( + tmp_path: Path, monkeypatch, +) -> None: + _, environment = _environment(tmp_path) + manifest = tmp_path / "synthetic" / "tests" / "manifest.json" + manifest.parent.mkdir(parents=True) + manifest.write_text( + json.dumps({"task": "text_translation", "reference_precision": "fp16"}), + encoding="utf-8", + ) + model = ModelDescriptor( + name="synthetic-translator", + hf_id="example/synthetic-translator", + hf_revision="a" * 40, + bundle_name="synthetic-translator.bundle", + family="synthetic", + task="text_translation", + precision="fp16", + manifest_path=manifest, + testcases=( + { + "name": "translate", + "source_text": "Hello", + "source_language": "eng_Latn", + "target_language": "fra_Latn", + "max_new_tokens": 8, + }, + ), + build_settings={"max_sequence_length": 32}, + ) + monkeypatch.setattr(perf.ManifestCatalog, "resolve", lambda *_args: model) + spec = { + "id": "synthetic.translate", + "family": "synthetic", + "operation": "translate", + "model": model.name, + "workload": {"testcase": "translate"}, + "measurement": {"warmup": 1, "iterations": 1}, + "baseline": { + "runner": "hf-transformers", + "task": "seq2seq-lm", + "mode": "hf-eager", + "timing_scope": "public_operation_call_wall", + "input_preparation_included": True, + "asset_loading_included": False, + "source_language_placement": "replace-final-unk", + }, + } + entry = perf.resolve_entries([spec], environment)[0] + + command = perf.baseline_command(entry, environment, tmp_path / "reference.json") + request = json.loads(command[command.index("--request-json") + 1]) + + assert request["source_language_placement"] == "replace-final-unk" + assert not any( + "source_language_placement" in argument + for argument in perf.candidate_command( + entry, environment, tmp_path / "candidate", no_build=True + ) + ) + + @pytest.mark.parametrize( "entry_id,task,operation", [ @@ -311,14 +402,14 @@ def test_reference_selection_does_not_rewrite_manifest_or_silently_choose_defaul manifest.write_text('{"task":"series_to_point_forecast"}') original = manifest.read_bytes() arguments = SimpleNamespace(manifest=manifest, selected_task="series_to_quantile_forecast") - assert task_reference._selected_task(arguments) == "series_to_quantile_forecast" + assert generic_reference._selected_task(arguments) == "series_to_quantile_forecast" assert manifest.read_bytes() == original arguments.selected_task = None - assert task_reference._selected_task(arguments) == "series_to_point_forecast" + assert generic_reference._selected_task(arguments) == "series_to_point_forecast" for invalid in ("", " ", " series_to_point_forecast", 7): arguments.selected_task = invalid with pytest.raises(ValueError, match="selected_task"): - task_reference._selected_task(arguments) + generic_reference._selected_task(arguments) @pytest.mark.parametrize("payload", [{"token_ids": []}, {"token_ids": [7], "prompt": "hello"}]) @@ -330,7 +421,7 @@ def test_text_only_reference_loaders_never_replace_token_input_with_empty_text(p "asset_loading_included": False, })) with pytest.raises(ValueError, match="token_ids"): - task_reference._load_embedding(arguments, payload, {}) + generic_reference._load_embedding(arguments, payload, {}) class _SummaryTensor: @@ -370,23 +461,23 @@ def tolist(self): def test_forecast_reference_names_axes_without_arbitrary_squeeze_or_sorting(): - point = task_reference._forecast_summary( + point = generic_reference._forecast_summary( _SummaryTensor(np.zeros((1, 4, 3))), "series_to_point_forecast" ) assert point["shape"] == [4, 3] and point["axes"] == ["horizon", "channel"] assert point["horizon_steps"] == [1, 2, 3, 4] and point["forecast_elements"] == 12 - quantile = task_reference._forecast_summary( + quantile = generic_reference._forecast_summary( _SummaryTensor(np.zeros((1, 3, 4))), "series_to_quantile_forecast", [0.1, 0.5, 0.9] ) assert quantile["shape"] == [3, 4, 1] assert quantile["axes"] == ["quantile", "horizon", "channel"] assert quantile["quantile_levels"] == [0.1, 0.5, 0.9] with pytest.raises(ValueError, match="one-series"): - task_reference._forecast_summary( + generic_reference._forecast_summary( _SummaryTensor(np.zeros((2, 4, 3))), "series_to_point_forecast" ) with pytest.raises(ValueError, match="quantile levels"): - task_reference._forecast_summary( + generic_reference._forecast_summary( _SummaryTensor(np.zeros((1, 3, 4))), "series_to_quantile_forecast", [] ) @@ -427,7 +518,7 @@ def test_semantic_forecast_comparison_rejects_swapped_or_missing_axes(): def test_regression_values_reference_retains_single_target_axis(): - summary = task_reference._regression_values_summary(_SummaryTensor([[1.5, -2.0]])) + summary = generic_reference._regression_values_summary(_SummaryTensor([[1.5, -2.0]])) assert summary["values"] == [1.5, -2.0] assert summary["axes"] == ["target"] and summary["target_count"] == 2 assert summary["target_names"] == [] and summary["target_units"] == [] @@ -445,9 +536,9 @@ def test_regression_values_reference_retains_single_target_axis(): {"output_summary": dict(invalid)})[0] for shape in ((2,), (2, 2), (1, 0), (1, 2, 1)): with pytest.raises(ValueError, match="one batch"): - task_reference._regression_values_summary(_SummaryTensor(np.zeros(shape))) + generic_reference._regression_values_summary(_SummaryTensor(np.zeros(shape))) with pytest.raises(ValueError, match="finite"): - task_reference._regression_values_summary(_SummaryTensor([[float("inf")]])) + generic_reference._regression_values_summary(_SummaryTensor([[float("inf")]])) def test_head_score_performance_contract_preserves_shape_and_representation(): @@ -486,7 +577,7 @@ def test_head_score_performance_contract_rejects_incomplete_outputs(changed): def test_regression_distribution_keeps_parameter_names_and_target_axis(): values = (_SummaryTensor([[1.0, 2.0]]), _SummaryTensor([[0.5, 0.7]])) - summary = task_reference._regression_summary(values, "normal", ["loc", "scale"]) + summary = generic_reference._regression_summary(values, "normal", ["loc", "scale"]) assert summary["target_count"] == 2 and summary["axes"] == ["target"] assert summary["parameters"] == [ {"name": "location", "values": [1.0, 2.0]}, @@ -502,10 +593,10 @@ def test_regression_distribution_keeps_parameter_names_and_target_axis(): {"output_summary": {**summary, "distribution": "student_t"}}, )[0] with pytest.raises(ValueError, match="same target axis"): - task_reference._regression_summary( + generic_reference._regression_summary( (_SummaryTensor([[1, 2]]), _SummaryTensor([[1]])), "normal", ["loc", "scale"] ) - student = task_reference._regression_summary( + student = generic_reference._regression_summary( (_SummaryTensor([[4.0]]), _SummaryTensor([[1.0]]), _SummaryTensor([[0.5]])), "student_t", ["df", "loc", "scale"], ) @@ -513,7 +604,7 @@ def test_regression_distribution_keeps_parameter_names_and_target_axis(): "degrees_of_freedom", "location", "scale" ] with pytest.raises(ValueError, match="parameter names"): - task_reference._regression_summary(values, "normal", ["loc", "location"]) + generic_reference._regression_summary(values, "normal", ["loc", "location"]) @pytest.mark.parametrize("distribution,parameters", [ @@ -569,635 +660,13 @@ def test_regression_distribution_order_fix_retains_mismatch_rejection(changed): def test_timeseries_reference_preserves_observed_masks_and_unobserved_padding(): request = {"past_values": [1, 2, 3], "observed_mask": [1, 0, 1]} - observed = task_reference._observed_values(request, 3) + observed = generic_reference._observed_values(request, 3) assert observed == [1.0, 0.0, 1.0] - assert task_reference._align(observed, 5, 0.0) == [0, 0, 1, 0, 1] - assert task_reference._observed_values({"past_values": [1, 2]}, 2) == [1, 1] - assert task_reference._observed_values({"observed_mask": []}, 2) == [1, 1] + assert generic_reference._align(observed, 5, 0.0) == [0, 0, 1, 0, 1] + assert generic_reference._observed_values({"past_values": [1, 2]}, 2) == [1, 1] + assert generic_reference._observed_values({"observed_mask": []}, 2) == [1, 1] with pytest.raises(ValueError, match="match past_values"): - task_reference._observed_values({"observed_mask": [1]}, 2) - - -def _voicechat_reference_fixture(tmp_path, monkeypatch): - events = [] - reference = tmp_path / "speech-reference" - utility = reference / "nemo/collections/speechlm2/inference/utils/offline_voicechat.py" - utility.parent.mkdir(parents=True) - utility.write_text("# official utility location fixture\n", encoding="utf-8") - checkpoint = tmp_path / "checkpoint" - checkpoint.mkdir() - (checkpoint / "config.json").write_text("{}", encoding="utf-8") - audio_path = tmp_path / "request.wav" - audio_path.write_bytes(b"fixture; decoded by the mocked official loader") - manifest = tmp_path / "manifest.json" - manifest.write_text('{"task":"offline_speech_dialogue"}', encoding="utf-8") - arguments = SimpleNamespace( - model=str(checkpoint), - revision="checkpoint-revision", - manifest=manifest, - family="nemotron_voicechat", - precision="fp32", - operation="speech_dialogue", - local_files_only=True, - ) - request = {"audio_path": str(audio_path)} - options = {"reference_repo": str(reference)} - state = { - "kwargs": [], - "prompts": [], - "events": events, - "metadata": SimpleNamespace(samplerate=16000, channels=1, frames=4), - } - - class Tensor(_SummaryTensor): - def __init__(self, values, label="output"): - super().__init__(values) - self.label = label - - def to(self, device): - events.append(("transfer", self.label, device)) - return self - - def __getitem__(self, key): - return Tensor(self.values[key], self.label) - - def cpu(self): - events.append(("materialize", self.label)) - return self - - def numpy(self): - return np.asarray(self.values, dtype=np.float32) - - class Model: - source_sample_rate = 16000 - target_sample_rate = 22050 - - def float(self): - events.append("float32") - return self - - model = Model() - signal = Tensor([[0.1, 0.2, 0.3, 0.4]], "signal") - lengths = Tensor([4], "lengths") - upstream = ModuleType("nemo.collections.speechlm2.inference.utils.offline_voicechat") - upstream.__file__ = str(utility) - - def build_model(path, *, device): - events.append(("build", path, device)) - return model - - def load_wav(path, *, device): - events.append(("decode", path, device)) - return signal, signal, lengths - - def encode_prompt(received_model, prompt, *, device): - assert received_model is model - events.append(("prompt", prompt, device)) - state["prompts"].append(prompt) - return "prompt-token-buffer", "prompt-length-buffer" - - def inference(received_model, **kwargs): - assert received_model is model - events.append("infer") - state["kwargs"].append(kwargs) - # The padded NaN must not be included in the valid audio result. - return state.get( - "result", - { - "text": ["Full spoken answer."], - "audio": Tensor([[0.1, 0.2, 0.3, float("nan")]]), - "audio_len": Tensor([3]), - }, - ) - - upstream.build_model = build_model - upstream.load_wav_16k_mono = load_wav - upstream.encode_system_prompt = encode_prompt - upstream.run_offline_inference = inference - sf = ModuleType("soundfile") - sf.info = lambda _path: state["metadata"] - - def write(path, values, rate, **kwargs): - events.append(("write_audio", str(path), rate, kwargs)) - state["written_audio"] = np.array(values, copy=True) - - sf.write = write - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setitem(sys.modules, upstream.__name__, upstream) - monkeypatch.setitem(sys.modules, "soundfile", sf) - monkeypatch.setitem(sys.modules, "torch", ModuleType("torch")) - monkeypatch.setattr( - task_reference, "_seed_all", lambda _torch, seed: events.append(("seed", seed)) - ) - monkeypatch.setattr(task_reference, "_synchronize", lambda: events.append("barrier")) - return arguments, request, options, state, upstream, Tensor - - -def test_voicechat_reference_builds_once_and_times_real_offline_calls(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - session = task_reference._load_voicechat(arguments, request, options) - assert session.timing_scope == "task-pipeline-call-wall" - assert session.input_preparation_included is True and session.asset_loading_included is False - assert state["events"] == [ - ("build", arguments.model, "cuda"), - "float32", - ("decode", request["audio_path"], "cpu"), - ] - samples, output = task_reference._measure(session, warmup=1, iterations=2) - assert len(samples) == 2 and all(value > 0 for value in samples) - assert sum(isinstance(event, tuple) and event[0] == "build" for event in state["events"]) == 1 - assert sum(isinstance(event, tuple) and event[0] == "decode" for event in state["events"]) == 1 - assert state["events"].count("infer") == 3 - assert state["events"].count(("seed", 0)) == 3 - assert state["events"].count(("transfer", "signal", "cuda")) == 3 - assert state["prompts"] == [task_reference.VOICECHAT_SYSTEM_PROMPT] * 3 - assert output["text"] == "Full spoken answer." - assert output["audio_samples"] == output["num_samples"] == 3 - assert output["sample_rate"] == 22050 and output["channels"] == 1 - assert output["input_samples"] == 4 and output["input_sample_rate"] == 16000 - np.testing.assert_array_equal( - output["_audio_f32"], np.asarray([0.1, 0.2, 0.3], dtype=np.float32) - ) - for kwargs in state["kwargs"]: - assert kwargs["decode_audio"] is True and kwargs["input_pad_len"] == 0 - assert kwargs["prompt_tokens"] == "prompt-token-buffer" - assert kwargs["prompt_token_lens"] == "prompt-length-buffer" - assert not ( - {"temperature", "top_p", "repetition_penalty", "function_calls"} & kwargs.keys() - ) - - -def test_voicechat_reference_preserves_explicit_prompt_sampling_and_seed(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - session = task_reference._load_voicechat( - arguments, - { - **request, - "system_prompt": " An explicit spoken prompt. ", - "config": { - "seed": 17, - "temperature": 0.0, - "top_p": 1.0, - "repetition_penalty": 1.2, - "presence_penalty": 0.0, - }, - }, - options, - ) - session.invoke() - kwargs = state["kwargs"][0] - assert state["prompts"] == [" An explicit spoken prompt. "] - assert ("seed", 17) in state["events"] - assert { - key: kwargs[key] - for key in ("temperature", "top_p", "repetition_penalty", "presence_penalty") - } == {"temperature": 0.0, "top_p": 1.0, "repetition_penalty": 1.2, "presence_penalty": 0.0} - - -@pytest.mark.parametrize( - "patch", - [ - {"tools": []}, - {"tail_frames": 1}, - {"finish_tail_frames": -1}, - {"max_new_tokens": 256}, - {"system_prompt": " "}, - {"seed": -1}, - {"temperature": True}, - ], -) -def test_voicechat_reference_rejects_unmatched_scope_before_loading(tmp_path, monkeypatch, patch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - with pytest.raises(ValueError): - task_reference._load_voicechat(arguments, {**request, **patch}, options) - assert state["events"] == [] - - -def test_voicechat_reference_does_not_relabel_live_dialogue_as_offline(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - arguments.manifest.write_text('{"task":"duplex_speech_dialogue"}', encoding="utf-8") - with pytest.raises(ValueError, match="live or tool"): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == [] - - -def test_voicechat_reference_uses_explicit_offline_task_without_changing_primary(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - arguments.manifest.write_text('{"task":"duplex_speech_dialogue"}', encoding="utf-8") - original = arguments.manifest.read_bytes() - arguments.selected_task = "offline_speech_dialogue" - session = task_reference._load_voicechat(arguments, request, options) - output = session.invoke() - assert state["kwargs"] and isinstance(output["text"], str) - assert output["audio_samples"] == output["num_samples"] - assert arguments.manifest.read_bytes() == original - for task in ("duplex_speech_dialogue", "tool_speech_dialogue", "unknown_task", ""): - arguments.selected_task = task - before = list(state["events"]) - with pytest.raises(ValueError): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == before - - -def test_voicechat_empty_prompt_matches_native_default_selection(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch) - session = task_reference._load_voicechat(arguments, {**request, "system_prompt": ""}, options) - session.invoke() - assert state["prompts"] == [task_reference.VOICECHAT_SYSTEM_PROMPT] - - -def test_voicechat_reference_rejects_unmatched_audio_and_stale_import(tmp_path, monkeypatch): - arguments, request, options, state, upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - state["metadata"] = SimpleNamespace(samplerate=48000, channels=2, frames=4) - with pytest.raises(ValueError, match="16 kHz mono"): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == [] - upstream.__file__ = str(tmp_path / "another-installation.py") - with pytest.raises(RuntimeError, match="different NeMo installation"): - task_reference._load_voicechat(arguments, request, options) - assert state["events"] == [] - - -def test_voicechat_reference_rejects_invalid_live_output(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - session = task_reference._load_voicechat(arguments, request, options) - state["result"] = { - "text": ["real text"], - "audio": tensor([[0.1, 0.2]]), - "audio_len": tensor([3]), - } - with pytest.raises(RuntimeError, match="audio_len"): - session.invoke() - state["result"] = { - "text": ["real text"], - "audio": tensor([[float("nan")]]), - "audio_len": tensor([1]), - } - with pytest.raises(RuntimeError, match="nonfinite"): - session.invoke() - - -def test_voicechat_runner_writes_audio_after_measurement(tmp_path, monkeypatch): - arguments, request, options, state, _upstream, _tensor = _voicechat_reference_fixture( - tmp_path, monkeypatch - ) - arguments.adapter = "nemo-voicechat" - arguments.request_json = json.dumps(request) - arguments.adapter_options_json = json.dumps(options) - arguments.timing_contract_json = json.dumps( - { - "timing_scope": "task-pipeline-call-wall", - "input_preparation_included": True, - "asset_loading_included": False, - } - ) - arguments.mode = "pytorch-eager" - arguments.padding = "longest" - arguments.warmup, arguments.iterations = 1, 2 - arguments.case_name = "offline-voicechat" - arguments.output = tmp_path / "reference.json" - monkeypatch.setattr(task_reference, "_environment", lambda: {}) - assert task_reference.run(arguments) == 0 - result = json.loads(arguments.output.read_text(encoding="utf-8")) - assert result["model_load_included"] is False - assert result["measurement_policy"]["input_preparation_included"] is True - assert result["output_summary"]["text"] == "Full spoken answer." - assert result["output_summary"]["audio_artifact"].endswith("reference.audio.wav") - assert "_audio_f32" not in result["output_summary"] - assert state["events"][-1] == ( - "write_audio", - str(tmp_path / "reference.audio.wav"), - 22050, - {"subtype": "FLOAT"}, - ) - np.testing.assert_array_equal( - state["written_audio"], np.asarray([0.1, 0.2, 0.3], dtype=np.float32) - ) - - -def _moge_reference_fixture(tmp_path, monkeypatch): - events = [] - reference = tmp_path / "moge-reference" - module_path = reference / "moge/model/v2.py" - module_path.parent.mkdir(parents=True) - module_path.write_text("# upstream location fixture\n", encoding="utf-8") - checkpoint = tmp_path / "checkpoint" - checkpoint.mkdir() - (checkpoint / "model.pt").write_bytes(b"fixture weights") - image_path = tmp_path / "input.png" - image_path.write_bytes(b"fixture decoded by mocked PIL") - manifest = tmp_path / "manifest.json" - manifest.write_text('{"task":"image_to_metric_geometry"}', encoding="utf-8") - arguments = SimpleNamespace( - model=str(checkpoint), - revision="checkpoint-revision", - manifest=manifest, - family="moge", - operation="geometry", - precision="fp32", - local_files_only=True, - ) - state = { - "events": events, - "inputs": [], - "inference": [], - "pixels": np.asarray([[[255, 0, 127], [0, 255, 64]]], dtype=np.uint8), - "checkpoint": {"model_config": {"encoder": "fixture"}, "model": {"weight": "fixture"}}, - "arrays": { - "points": np.asarray([[[0.25, -0.5, 2], [np.inf, np.inf, np.inf]]], dtype=np.float32), - "depth": np.asarray([[2, np.inf]], dtype=np.float32), - "mask": np.asarray([[True, False]]), - "intrinsics": np.asarray([[0.8, 0, 0.5], [0, 1.1, 0.5], [0, 0, 1]], dtype=np.float32), - }, - } - - class Tensor: - def __init__(self, values): - self.values = np.asarray(values) - - def permute(self, *axes): - events.append(("permute", axes)) - return Tensor(self.values.transpose(axes)) - - def to(self, device): - events.append(("input_transfer", device)) - return self - - def detach(self): - return self - - def cpu(self): - events.append("materialize") - return self - - def numpy(self): - return self.values - - class Model: - def __init__(self, **kwargs): - events.append(("construct", kwargs)) - state["model"] = self - - def load_state_dict(self, weights, *, strict): - events.append(("load_state", weights, strict)) - return state.get("state_mismatch", ([], [])) - - def eval(self): - events.append("eval") - return self - - def float(self): - events.append("float32") - return self - - def to(self, device): - events.append(("model_transfer", device)) - return self - - def infer(self, tensor, **kwargs): - events.append("infer") - state["inputs"].append(np.array(tensor.values, copy=True)) - state["inference"].append(kwargs) - return {name: Tensor(value) for name, value in state["arrays"].items()} - - module = ModuleType("moge.model.v2") - module.__file__ = str(module_path) - module.MoGeModel = Model - attention = ModuleType("torch.nn.attention") - attention.SDPBackend = SimpleNamespace(MATH=object()) - - class MathContext: - def __enter__(self): - events.append("math_enter") - - def __exit__(self, *_args): - events.append("math_exit") - - def sdpa_kernel(backends): - assert backends == [attention.SDPBackend.MATH] - events.append("math_backend") - return MathContext() - - attention.sdpa_kernel = sdpa_kernel - fake_torch = ModuleType("torch") - - def load(path, **kwargs): - events.append(("load_checkpoint", str(path), kwargs)) - return state["checkpoint"] - - def from_numpy(values): - events.append("prepare_tensor") - return Tensor(values) - - fake_torch.load = load - fake_torch.from_numpy = from_numpy - fake_torch.backends = SimpleNamespace( - cuda=SimpleNamespace(matmul=SimpleNamespace(allow_tf32=True)), - cudnn=SimpleNamespace(allow_tf32=True), - ) - - class Image: - def __enter__(self): - return self - - def __exit__(self, *_args): - events.append("close_image") - - def convert(self, mode): - events.append(("color", mode)) - return self - - def __array__(self, dtype=None, copy=None): - return np.asarray(state["pixels"], dtype=dtype) - - def open_image(path): - events.append(("decode_image", str(path))) - return Image() - - pil = ModuleType("PIL") - pil.Image = SimpleNamespace(open=open_image) - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setitem(sys.modules, "torch", fake_torch) - monkeypatch.setitem(sys.modules, "PIL", pil) - monkeypatch.setitem(sys.modules, module.__name__, module) - monkeypatch.setitem(sys.modules, attention.__name__, attention) - monkeypatch.setattr(task_reference, "_synchronize", lambda: events.append("barrier")) - return ( - arguments, - {"image_path": str(image_path)}, - {"reference_repo": str(reference), "num_tokens": 1800}, - state, - ) - - -def test_moge_reference_uses_public_upstream_apis_and_loads_once(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - session = task_reference._load_moge(arguments, request, options) - assert session.timing_scope == "task-pipeline-call-wall" - assert session.input_preparation_included and not session.asset_loading_included - assert ( - "load_checkpoint", - str(Path(arguments.model) / "model.pt"), - {"map_location": "cpu", "weights_only": True, "mmap": True}, - ) in state["events"] - assert state["model"].onnx_compatible_mode is True - assert "prepare_tensor" not in state["events"] and "infer" not in state["events"] - samples, summary = task_reference._measure(session, warmup=1, iterations=2) - assert len(samples) == 2 and all(value > 0 for value in samples) - assert ( - sum(isinstance(event, tuple) and event[0] == "load_checkpoint" for event in state["events"]) - == 1 - ) - assert ( - sum(isinstance(event, tuple) and event[0] == "decode_image" for event in state["events"]) - == 1 - ) - assert state["events"].count("prepare_tensor") == 3 - assert ( - state["events"].count("infer") - == state["events"].count("math_backend") - == state["events"].count("math_enter") - == state["events"].count("math_exit") - == 3 - ) - for controls in state["inference"]: - assert controls == { - "num_tokens": 1800, - "use_fp16": False, - "force_projection": True, - "apply_mask": True, - } - expected = (state["pixels"].astype(np.float32) / 255.0).transpose(2, 0, 1) - for values in state["inputs"]: - np.testing.assert_array_equal(values, expected) - assert summary["point_shape"] == [1, 2, 3] and summary["units"] == "meters" - assert summary["requested_fov_x"] is None - assert np.isposinf(summary["_geometry_arrays"]["depth"][0, 1]) - - -def test_moge_reference_preserves_explicit_fov_degrees(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - session = task_reference._load_moge(arguments, {**request, "config": {"fov_x": 72.5}}, options) - summary = session.invoke() - assert state["inference"][0]["fov_x"] == 72.5 - assert summary["requested_fov_x"] == 72.5 - - -@pytest.mark.parametrize( - "request_update,options_update", - [ - ({}, {"num_tokens": None}), - ({"num_tokens": 3600}, {}), - ({"num_tokens": 1800}, {"num_tokens": 3600}), - ({"fov_x": 0}, {}), - ({"fov_x": 180}, {}), - ({"fov_x": True}, {}), - ({"apply_mask": False}, {}), - ], -) -def test_moge_reference_rejects_unqualified_controls_before_model_load( - tmp_path, monkeypatch, request_update, options_update -): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - with pytest.raises(ValueError): - task_reference._load_moge( - arguments, {**request, **request_update}, {**options, **options_update} - ) - assert not any( - isinstance(event, tuple) and event[0] == "load_checkpoint" for event in state["events"] - ) - - -def test_moge_reference_preserves_strict_checkpoint_contract(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - state["state_mismatch"] = (["uninitialized_weight"], []) - with pytest.raises(ValueError, match="state mismatch"): - task_reference._load_moge(arguments, request, options) - assert "infer" not in state["events"] - state["checkpoint"]["unexpected"] = 1 - with pytest.raises(ValueError, match="top-level"): - task_reference._load_moge(arguments, request, options) - - -def test_moge_geometry_artifacts_preserve_masked_infinity_and_calibration(tmp_path, monkeypatch): - arguments, request, options, state = _moge_reference_fixture(tmp_path, monkeypatch) - summary = dict(task_reference._load_moge(arguments, request, options).invoke()) - task_reference._write_geometry_artifacts(summary, tmp_path / "reference.json") - assert "_geometry_arrays" not in summary - assert summary["height"] == 1 and summary["width"] == 2 and summary["valid_pixels"] == 1 - np.testing.assert_array_equal( - np.fromfile(summary["points_artifact"], dtype=" tuple[Path, perf.Environment]: @@ -1216,8 +685,14 @@ def _environment(tmp_path: Path) -> tuple[Path, perf.Environment]: "libtrtmc_model_lance.so", ): (runtime / name).write_bytes(b"") + _, suite_entries, _ = perf._load_suite_file(SUITE) + reference_fields = { + str(declaration["environment"]) + for entry in suite_entries + for declaration in entry.get("baseline", {}).get("reference_inputs", {}).values() + } references = {} - for name in perf.REFERENCE_FIELDS: + for name in reference_fields: path = tmp_path / name path.mkdir() references[name] = str(path) @@ -1393,7 +868,7 @@ def test_shared_hf_cache_cannot_be_deleted(tmp_path: Path) -> None: def test_checked_in_environments_have_no_dead_gpu_headroom_setting() -> None: - root = REPO / "apps/benchmark/performance/environments" + root = REPO / "qualification_tests/benchmark_qualification/performance/config/environments" for path in root.glob("*.yaml"): value = yaml.safe_load(path.read_text(encoding="utf-8")) assert "minimum_gpu_free_fraction" not in value["execution"], path @@ -1434,13 +909,16 @@ def test_release_suite_expands_profiles_and_covers_ready_catalog() -> None: vision_entries = {entry["id"]: entry for entry in entries if entry["id"] in vision_ids} assert set(vision_entries) == vision_ids perf._coverage(entries, excluded) - # Adapter identity belongs to the builtin fixture, not an owned replacement. + # Adapter identity belongs to the suite entry, not a shared family-name registry. _, builtin_entries, _ = perf._load_suite_file(SUITE) builtin_vision = {entry["id"]: entry for entry in builtin_entries if entry["id"] in vision_ids} assert set(builtin_vision) == vision_ids + builtin_timm = [entry for entry in builtin_entries if entry["family"].startswith("timm_")] + assert builtin_timm assert all( - entry["baseline"]["adapter"] == "hf-transformers-vision" - for entry in builtin_vision.values() + entry["baseline"]["adapter"] == "timm-classification" + and entry["baseline"]["reference_backend"] == "timm" + for entry in builtin_timm ) @@ -1449,16 +927,83 @@ def test_release_suite_expands_profiles_and_covers_ready_catalog() -> None: "family", "expected_scope", "input_preparation_included", + "mode", "calls_after_load", "calls_after_invoke", + "calls_after_summary", + "compiled", ), [ - ("bert", "task-pipeline-call-wall", True, [], ["tokenize", "model"]), - ("eagle_vlm", "task-model-call-wall", False, ["tokenize"], ["tokenize", "model"]), - ("bert", "task-model-call-wall", False, ["tokenize"], ["tokenize", "model"]), - ("eagle_vlm", "task-pipeline-call-wall", True, [], ["tokenize", "model"]), - ("renamed_embedding", "task-model-call-wall", False, ["tokenize"], ["tokenize", "model"]), - ("renamed_embedding", "task-pipeline-call-wall", True, [], ["tokenize", "model"]), + ( + "bert", + "task-pipeline-call-wall", + True, + "hf-eager", + [], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "bert", + "task-pipeline-call-wall", + True, + "torch-compile", + ["compile"], + ["compile", "tokenize", "model", "materialize"], + ["compile", "tokenize", "model", "materialize", "validate"], + True, + ), + ( + "eagle_vlm", + "task-model-call-wall", + False, + "hf-eager", + ["tokenize"], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "bert", + "task-model-call-wall", + False, + "hf-eager", + ["tokenize"], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "eagle_vlm", + "task-pipeline-call-wall", + True, + "hf-eager", + [], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "renamed_embedding", + "task-model-call-wall", + False, + "hf-eager", + ["tokenize"], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), + ( + "renamed_embedding", + "task-pipeline-call-wall", + True, + "hf-eager", + [], + ["tokenize", "model", "materialize"], + ["tokenize", "model", "materialize", "validate"], + False, + ), ], ) def test_embedding_reference_measures_the_declared_timing_contract( @@ -1466,8 +1011,11 @@ def test_embedding_reference_measures_the_declared_timing_contract( family, expected_scope, input_preparation_included, + mode, calls_after_load, calls_after_invoke, + calls_after_summary, + compiled, ) -> None: calls: list[str] = [] @@ -1478,6 +1026,10 @@ class FakeTensor: def to(self, *_args, **_kwargs): return self + def detach(self): + calls.append("materialize") + return self + def unsqueeze(self, _dimension): return self @@ -1491,6 +1043,7 @@ def numel(self): return 2 def isfinite(self): + calls.append("validate") return self def all(self): @@ -1546,9 +1099,16 @@ def __call__(self, **_kwargs): fake_transformers.AutoTokenizer = FakeTokenizer monkeypatch.setitem(sys.modules, "torch", fake_torch) monkeypatch.setitem(sys.modules, "transformers", fake_transformers) + compile_evidence = {"compiled_graph_count": 0} + monkeypatch.setattr( + generic_reference, + "_compile_forward", + lambda _model: calls.append("compile") or compile_evidence, + ) arguments = SimpleNamespace( family=family, model="sentence-transformers/all-MiniLM-L6-v2", + mode=mode, precision="fp32", revision="model-revision", trust_remote_code=False, @@ -1560,7 +1120,7 @@ def __call__(self, **_kwargs): }), ) - session = task_reference.LOADERS["hf-transformers-embedding"]( + session = generic_reference.LOADERS["hf-transformers-embedding"]( arguments, {"prompt": "The quick brown fox"}, {}, @@ -1570,8 +1130,42 @@ def __call__(self, **_kwargs): assert session.timing_scope == expected_scope assert session.input_preparation_included is input_preparation_included assert session.asset_loading_included is False - assert session.invoke()["embedding_vectors"] == 1 + assert (session.compile_evidence is compile_evidence) is compiled + vector = session.invoke() assert calls == calls_after_invoke + assert session.summarize is not None + assert session.summarize(vector)["embedding_vectors"] == 1 + assert calls == calls_after_summary + + +@pytest.mark.parametrize(("compiled", "expected_invocations"), [(False, 2), (True, 3)]) +def test_zero_warmup_keeps_compilation_outside_timed_samples( + monkeypatch, compiled, expected_invocations +) -> None: + invocations = 0 + compile_evidence = {"compiled_graph_count": 0} if compiled else None + + def invoke(): + nonlocal invocations + invocations += 1 + if compile_evidence is not None and invocations == 1: + compile_evidence["compiled_graph_count"] = 1 + return {"value": invocations} + + monkeypatch.setattr(generic_reference, "_synchronize", lambda: None) + samples, output = generic_reference._measure( + generic_reference.Session( + invoke=invoke, + framework="test", + compile_evidence=compile_evidence, + ), + warmup=0, + iterations=2, + ) + + assert len(samples) == 2 + assert invocations == expected_invocations + assert output == {"value": expected_invocations} def test_check_resolves_selected_entry_with_one_runtime_root(tmp_path: Path, capsys) -> None: @@ -1628,7 +1222,7 @@ def run_command(arguments, **kwargs): return successful(arguments, **kwargs) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -1664,7 +1258,7 @@ def test_compiled_output_mismatch_tries_the_configured_fallback( monkeypatch.setattr(perf, "run_command", successful) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -1684,39 +1278,6 @@ def test_compiled_output_mismatch_tries_the_configured_fallback( assert row["reference_attempts"][0]["fallback_reason"] == "output contract mismatch" -def test_lerobot_reference_is_family_owned_and_has_a_closed_contract(tmp_path: Path) -> None: - _, environment = _environment(tmp_path) - source = Path(environment.references["lerobot_repo"]) - entrypoint = source / "lerobot/common/policies/act/modeling_act.py" - entrypoint.parent.mkdir(parents=True) - entrypoint.write_text("", encoding="utf-8") - # Exercise the builtin adapter's parser and source-root contract explicitly. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "lerobot_act.control"] - resolved = perf.resolve_entries(selected, environment)[0] - command = perf.baseline_command(resolved, environment, tmp_path / "reference.json") - parsed = task_reference.build_parser().parse_args(command[2:]) - assert parsed.adapter == "pytorch-lerobot-act" - assert json.loads(parsed.adapter_options_json) == {"source_root": str(source)} - - candidate = { - "action_steps": 100, - "action_dim": 14, - "action_values": 1400, - "within_training_bounds": True, - } - reference = { - "action_steps": 100, - "action_dim": 14, - "action_values": 1400, - "finite": True, - } - assert perf._output_contract( - resolved, - {"output_summary": candidate}, - {"output_summary": reference}, - ) == (True, "", None) - def test_comparison_preserves_output_gate_and_three_performance_states( tmp_path: Path, @@ -1787,7 +1348,7 @@ def test_unstable_measurement_is_retried_once( candidate_samples=(falling, second), ) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -1825,7 +1386,7 @@ def test_scratch_is_run_scoped_and_success_cleans_all_entry_attempts( state, run_command = _fake_measurement_runner(environment, entry) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, run, @@ -1850,7 +1411,7 @@ def test_existing_artifact_attempt_is_skipped_in_one_execution(tmp_path: Path, m _, run_command = _fake_measurement_runner(environment, entry) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, run, @@ -1885,7 +1446,7 @@ def execute(_entry, _environment, _run, *, attempt, **_kwargs): "selected_entry_ids": ["first"], "rows": [], } - monkeypatch.setattr(perf, "_execute_entry", execute) + monkeypatch.setattr(perf, "execute_entry", execute) monkeypatch.setattr(perf, "_write_json", lambda *_args, **_kwargs: None) monkeypatch.setattr(perf, "write_report", lambda *_args, **_kwargs: {}) @@ -1933,7 +1494,7 @@ def test_second_measurement_contract_mismatch_discards_first_stability( ) monkeypatch.setattr(perf, "run_command", run_command) - row = perf._execute_entry( + row = perf.execute_entry( entry, environment, tmp_path / "run", @@ -1965,7 +1526,7 @@ def test_failed_remeasurement_is_not_treated_as_a_pass(tmp_path: Path, monkeypat monkeypatch.setattr(perf, "run_command", run_command) with pytest.raises(perf.PerfMatrixError, match="candidate command failed"): - perf._execute_entry( + perf.execute_entry( entry, environment, tmp_path / "run", @@ -1995,7 +1556,7 @@ def test_delete_always_cleans_declared_bundle_when_candidate_process_fails( monkeypatch.setattr(perf, "run_command", run_command) with pytest.raises(perf.PerfMatrixError, match="candidate command failed"): - perf._execute_entry( + perf.execute_entry( entry, environment, tmp_path / "run", @@ -2019,7 +1580,7 @@ def test_delete_always_cleans_declared_bundle_when_candidate_process_fails( ) monkeypatch.setattr(perf, "run_command", run_command) with pytest.raises(perf.PerfMatrixError, match="candidate command failed"): - perf._execute_entry( + perf.execute_entry( external_entry, environment, tmp_path / "external-run", @@ -2126,7 +1687,7 @@ def write_json(path, value): if path.name == "results.json": snapshots.append([row["id"] for row in value["rows"]]) - monkeypatch.setattr(perf, "_execute_entry", execute) + monkeypatch.setattr(perf, "execute_entry", execute) monkeypatch.setattr(perf, "_write_json", write_json) monkeypatch.setattr(perf, "write_report", lambda *_args, **_kwargs: {}) @@ -2154,7 +1715,7 @@ def test_contract_mismatch_is_finished_but_keeps_run_non_green(tmp_path: Path, m } monkeypatch.setattr( perf, - "_execute_entry", + "execute_entry", lambda *_args, **_kwargs: pytest.fail("finished contract mismatch was rerun"), ) monkeypatch.setattr(perf, "_write_json", lambda *_args, **_kwargs: None) @@ -2285,65 +1846,18 @@ def run_command(arguments, *, stdout_path, stderr_path, **_kwargs): def test_reference_runner_dependencies_are_baseline_owned() -> None: - root = REPO / "apps/benchmark/performance/baselines" + root = REPO / "qualification_tests/benchmark_qualification/performance/references" required = { "audio_reference.py", - "elf_reference.py", - "lance_reference.py", - "reference_support.py", - "sana_wm_reference.py", } assert all((root / name).is_file() for name in required) - source = (root / "task_reference.py").read_text(encoding="utf-8") + source = (root / "generic_reference.py").read_text(encoding="utf-8") assert "from tools." not in source assert 'REPOSITORY / "tools/' not in source assert "tests/e2e" not in source assert "tensorrt_model_connect.families" not in source -def test_lance_command_requires_explicit_checkout_without_a_commit_gate( - tmp_path: Path, -) -> None: - _, environment = _environment(tmp_path) - checkout = Path(environment.references["lance_repo"]) - (checkout / "inference_lance.py").write_text("", encoding="utf-8") - # This checkout requirement belongs to the existing builtin Lance adapter. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "lance.generate"] - resolved = perf.resolve_entries(selected, environment)[0] - command = perf.baseline_command(resolved, environment, tmp_path / "reference.json") - parsed = task_reference.build_parser().parse_args(command[2:]) - options = json.loads(parsed.adapter_options_json) - assert parsed.adapter == "upstream-lance" - assert options == { - "model_subdir": "Lance_3B", - "reference_repo": str(checkout), - "resolution": "image_768res", - "vit_subdir": "Qwen2.5-VL-ViT", - } - - direct = lance_reference.build_parser().parse_args( - [ - "--reference-repo", - str(checkout), - "--model", - "model", - "--image", - str(tmp_path / "image.png"), - "--prompt", - "describe", - "--max-new-tokens", - "4", - "--warmup", - "1", - "--iterations", - "2", - "--output", - str(tmp_path / "output.json"), - ] - ) - assert direct.reference_repo == checkout - def test_check_fails_fast_when_selected_reference_input_is_missing( tmp_path: Path, monkeypatch, capsys @@ -2392,7 +1906,7 @@ def test_timeseries_entries_use_current_forecast_request_schema(tmp_path: Path) assert not ({"branch_input", "field_input", "trunk_input"} & entry.case.request.keys()) timesfm = next(entry for entry in resolved if entry.model.family == "timesfm") assert timesfm.case.request["frequency"] == 2 - source = (REPO / "apps/benchmark/performance/baselines/task_reference.py").read_text( + source = (REPO / "qualification_tests/benchmark_qualification/performance/references/generic_reference.py").read_text( encoding="utf-8" ) assert '_numeric_values(request, "past_values")' in source @@ -2400,598 +1914,6 @@ def test_timeseries_entries_use_current_forecast_request_schema(tmp_path: Path) assert '_numeric_values(request, "field_input")' not in source -def test_qwen3_omni_uses_the_text_generation_contract(tmp_path: Path) -> None: - _, environment = _environment(tmp_path) - # Check the builtin text-only adapter, independently of the active owned route. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "qwen3_omni.generate"] - resolved = perf.resolve_entries(selected, environment)[0] - assert resolved.spec["operation"] == "generate" - assert resolved.spec["baseline"]["output_contract"] == "exact-text" - assert resolved.case.request["max_new_tokens"] == 16 - assert "talker_max_new_tokens" not in resolved.case.request - command = perf.baseline_command(resolved, environment, tmp_path / "reference.json") - assert command[command.index("--adapter") + 1] == "hf-qwen3-omni" - assert command[command.index("--operation") + 1] == "generate" - request = json.loads(command[command.index("--request-json") + 1]) - assert request["max_new_tokens"] == 16 - assert "talker_max_new_tokens" not in request - source = (REPO / "apps/benchmark/performance/baselines/task_reference.py").read_text( - encoding="utf-8" - ) - assert '"enable_audio_output": False' in source - assert "return_audio=False" in source - - -def test_sana_reference_reports_materialized_video_shape() -> None: - video = np.stack( - [ - np.zeros((24, 32, 3), dtype=np.uint8), - np.ones((24, 32, 3), dtype=np.uint8), - ] - ) - summary = sana_wm_reference.media_summary(video) - assert summary == { - "media_type": "video", - "media_count": 2, - "num_frames": 2, - "height": 24, - "width": 32, - "channels": 3, - } - assert perf._media_shape(summary) == (2, 24, 32, 3) - source = (REPO / "apps/benchmark/performance/baselines/task_reference.py").read_text( - encoding="utf-8" - ) - assert 'request.get("action"' in source - - -def test_sana_world_request_preserves_official_camera_controls(tmp_path: Path) -> None: - model = replace( - perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional"), - task="world_model_generation", - ) - request = perf.resolve_case( - model, tmp_path / "model.bundle", selected_task="world_model_generation", - ).request - assert request["translation_speed"] == 0.055 - assert request["rotation_speed_deg"] == 1.2 - assert request["fps"] == 16 - assert request["flow_shift"] == 9.8 - assert request["no_action_overlay"] is True - - -@pytest.mark.parametrize("route", ["builtin", "script", "hf"]) -def test_reference_testcase_name_is_only_sent_to_builtin_runner(monkeypatch, tmp_path, route): - _, environment = _environment(tmp_path) - # Exercise explicit runner fixtures, not a family's active owned replacement. - _, entries, _ = perf._load_suite_file(SUITE) - entry = perf.resolve_entries( - [value for value in entries if value["id"] == "sana_wm.generate_image"], environment, - )[0] - baseline = dict(entry.spec["baseline"]) - if route == "script": - baseline.pop("adapter") - baseline["script"] = "tests/reference.py" - monkeypatch.setattr(perf, "_family_script", lambda _: tmp_path / "reference.py") - elif route == "hf": - baseline["runner"] = "hf-transformers" - entry = replace(entry, spec={**entry.spec, "baseline": baseline}) - command = perf.baseline_command(entry, environment, tmp_path / "reference.json") - assert command[command.index("--case-name") + 1] == entry.spec["id"] - if route == "builtin": - assert command[command.index("--testcase-name") + 1] == entry.case.testcase_name - parsed = task_reference.build_parser().parse_args(command[2:]) - assert parsed.testcase_name == entry.case.testcase_name - index = command.index("--testcase-name") - direct = command[2:index] + command[index + 2:] - assert task_reference.build_parser().parse_args(direct).testcase_name is None - else: - assert "--testcase-name" not in command - - -@pytest.mark.parametrize("selection", ["selected", "missing", "duplicate", ""]) -def test_sana_reference_resolves_only_exact_testcase_before_execution( - monkeypatch, tmp_path, selection, -): - manifest = tmp_path / "manifest.json" - controls = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - chosen = {"name": "selected", **controls} - manifest.write_text(json.dumps({"video_num_frames": 321, "testcases": [ - {"name": "performance-entry", **dict.fromkeys(controls, "wrong testcase")}, - chosen, *([chosen] if selection == "duplicate" else []), - ]})) - arguments = SimpleNamespace( - manifest=manifest, testcase_name="selected" if selection == "duplicate" else selection, - case_name="performance-entry", warmup=1, iterations=2, - ) - assets = REPO / "families/sana_wm/tests/assets" - request = {"prompt": "drive forward", "image_path": str(assets / "demo_0.png"), - "action": "w-320", "num_steps": 60, "cfg_scale": 5.0, "seed": 42} - options = {"reference_repo": str(tmp_path), "model_dir": str(tmp_path / "model"), - "intrinsics": str(assets / "demo_0_intrinsics.npy")} - before = deepcopy(request), deepcopy(options), manifest.read_bytes() - - def run(command, **kwargs): - assert selection == "selected", "invalid selection must fail before reference execution" - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(controls[name]) - assert "--no_action_overlay" in command - assert command[command.index("--num_frames") + 1] == "321" - raise RuntimeError("captured selected reference command") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - if selection == "selected": - with pytest.raises(RuntimeError, match="captured selected reference command"): - task_reference._run_sana_wm(arguments, request, options) - else: - with pytest.raises(ValueError, match="exactly one testcase"): - task_reference._run_sana_wm(arguments, request, options) - assert (request, options, manifest.read_bytes()) == before - - -def test_sana_reference_options_use_resolved_testcase_and_explicit_options( - monkeypatch, tmp_path: Path, -) -> None: - _, environment = _environment(tmp_path) - # This tests the existing builtin adapter even after an owned script takes over. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] - entry = perf.resolve_entries(selected, environment)[0] - original = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - assert entry.case.testcase_name != entry.spec["id"] - testcase = next(value for value in entry.manifest["testcases"] - if value["name"] == entry.case.testcase_name) - other = {**testcase, "name": entry.spec["id"], **dict.fromkeys(original, "wrong testcase")} - entry = replace(entry, manifest={**entry.manifest, "testcases": [other, testcase]}) - semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") - request = task_reference.flatten_config(semantic.request) - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text(json.dumps(entry.manifest)) - arguments = SimpleNamespace( - manifest=manifest_path, testcase_name=entry.case.testcase_name, warmup=1, iterations=2, - ) - before = json.dumps(entry.manifest, sort_keys=True) - options = perf._adapter_options(entry, environment) - assert not original.keys() & options.keys() - assert "action" not in options and "prompt" not in options - assert options["reference_repo"] == str(Path(environment.references["sana_repo"]).resolve()) - assert options["model_dir"] == str(Path(environment.references["sana_model"]).resolve()) - options["intrinsics"] = str(entry.model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy") - expected = original - - def run(command, **kwargs): - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(expected[name]) - assert ("--no_action_overlay" in command) is expected["no_action_overlay"] - assert command[command.index("--action") + 1] == request["action"] - assert Path(command[command.index("--prompt") + 1]).read_text() == request["prompt"] - raise RuntimeError("captured reference options") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - original_options = deepcopy(options) - with pytest.raises(RuntimeError, match="captured reference options"): - task_reference._run_sana_wm(arguments, request, options) - assert options == original_options - explicit = {"translation_speed": 0.0, "rotation_speed_deg": 0.0, - "fps": 0, "flow_shift": 0.0, "no_action_overlay": False} - configured = {**entry.spec["baseline"].get("adapter_options", {}), **explicit} - entry = replace(entry, spec={**entry.spec, "baseline": {**entry.spec["baseline"], "adapter_options": configured}}) - options = perf._adapter_options(entry, environment) - assert {name: options[name] for name in original} == explicit - assert options["no_action_overlay"] is False - options["intrinsics"] = original_options["intrinsics"] - explicit_options = deepcopy(options) - expected = explicit - with pytest.raises(RuntimeError, match="captured reference options"): - task_reference._run_sana_wm(arguments, request, options) - assert options == explicit_options - assert request == task_reference.flatten_config(semantic.request) - assert json.loads(manifest_path.read_text()) == entry.manifest - assert json.dumps(entry.manifest, sort_keys=True) == before - assert entry.spec["baseline"]["adapter_options"] == configured - - -def test_sana_semantic_request_metadata_moves_only_to_reference_options( - monkeypatch, tmp_path: Path, -) -> None: - _, environment = _environment(tmp_path) - # Keep the builtin reference contract independent of canonical script selection. - _, entries, _ = perf._load_suite_file(SUITE) - selected = [entry for entry in entries if entry["id"] == "sana_wm.generate_image"] - entry = perf.resolve_entries(selected, environment)[0] - semantic = perf.resolve_case(entry.model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") - entry = replace(entry, case=semantic) - options = perf._adapter_options(entry, environment) - expected = {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - assert not expected.keys() & semantic.request.keys() - assert not expected.keys() & options.keys() - command = perf.baseline_command(entry, environment, tmp_path / "reference.json") - arguments = task_reference.build_parser().parse_args(command[2:]) - - def run(command, **kwargs): - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(expected[name]) - assert "--no_action_overlay" in command - raise RuntimeError("captured reference metadata") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - with pytest.raises(RuntimeError, match="captured reference metadata"): - task_reference._run_sana_wm(arguments, task_reference.flatten_config(semantic.request), options) - assert semantic.request["action"] == entry.manifest["testcases"][0]["action"] - assert semantic.request["seed"] == 42 and semantic.request["num_steps"] == 60 - assert semantic.request["cfg_scale"] == 5.0 - - -def test_sana_reference_calls_official_pipeline_with_exact_workload( - monkeypatch, tmp_path: Path -) -> None: - captured: dict[str, object] = {"generate": []} - reference_repo = tmp_path / "Sana" - reference_repo.mkdir() - model_dir = tmp_path / "model" - (model_dir / "dit").mkdir(parents=True) - (model_dir / "refiner/text_encoder").mkdir(parents=True) - (model_dir / "config.yaml").write_text("{}\n", encoding="utf-8") - (model_dir / "dit/sana_wm_1600m_720p.safetensors").write_bytes(b"weights") - image = tmp_path / "image.png" - image.write_bytes(b"image") - prompt = tmp_path / "prompt.txt" - prompt.write_text("drive forward", encoding="utf-8") - intrinsics = tmp_path / "intrinsics.npy" - intrinsics.write_bytes(b"intrinsics") - output = tmp_path / "result.json" - - class FakeImage: - def convert(self, mode): - captured["image_mode"] = mode - return self - - pil = ModuleType("PIL") - pil.Image = SimpleNamespace(open=lambda path: FakeImage()) - monkeypatch.setitem(sys.modules, "PIL", pil) - - synchronize_calls = [] - torch = ModuleType("torch") - torch.cuda = SimpleNamespace( - is_available=lambda: True, - synchronize=lambda: synchronize_calls.append(True), - ) - monkeypatch.setitem(sys.modules, "torch", torch) - - pyrallis = ModuleType("pyrallis") - - def parse_config(**kwargs): - captured["parse"] = kwargs - return "config" - - pyrallis.parse = parse_config - monkeypatch.setitem(sys.modules, "pyrallis", pyrallis) - - class RefinerSettings: - def __init__(self, **kwargs): - captured["refiner"] = kwargs - - class GenerationParams: - def __init__(self, **kwargs): - captured["generation"] = kwargs - - class Pipeline: - def __init__(self, **kwargs): - captured["pipeline"] = kwargs - - def generate(self, *args): - captured["generate"].append(args) - return { - "video": np.zeros((321, 24, 32, 3), dtype=np.uint8), - "c2w": "camera", - } - - trajectory = np.zeros((321, 4, 4), dtype=np.float32) - official = SimpleNamespace( - InferenceConfig=object, - RefinerSettings=RefinerSettings, - GenerationParams=GenerationParams, - SanaWMPipeline=Pipeline, - action_string_to_c2w=lambda action, **kwargs: ( - captured.update(action=(action, kwargs)) or trajectory - ), - _snap_num_frames=lambda value, **kwargs: value, - resize_and_center_crop=lambda value: ("cropped", (1, 1), (2, 2), (0, 0)), - load_intrinsics=lambda path, frames: ( - captured.update(load_intrinsics=(path, frames)) or "raw-intrinsics" - ), - transform_intrinsics_for_crop=lambda value, *sizes: ( - captured.update(transform_intrinsics=(value, sizes)) or "intrinsics" - ), - apply_overlay=lambda *_: (_ for _ in ()).throw( - AssertionError("no-action-overlay must skip overlay") - ), - ) - monkeypatch.setattr(sana_wm_reference, "_official_module", lambda path: official) - monkeypatch.setattr( - sys, - "argv", - [ - "sana_wm_reference.py", - "--reference-repo", - str(reference_repo), - "--image", - str(image), - "--model-dir", - str(model_dir), - "--prompt", - str(prompt), - "--action", - "w-320", - "--intrinsics", - str(intrinsics), - "--num_frames", - "321", - "--fps", - "16", - "--step", - "60", - "--cfg_scale", - "5.0", - "--flow_shift", - "9.8", - "--seed", - "42", - "--refiner_seed", - "42", - "--translation_speed", - "0.055", - "--rotation_speed_deg", - "1.2", - "--no_action_overlay", - "--warmup", - "1", - "--iterations", - "2", - "--output", - str(output), - ], - ) - - assert sana_wm_reference.main() == 0 - assert captured["action"] == ( - "w-320", - {"translation_speed": 0.055, "rotation_speed_deg": 1.2}, - ) - assert captured["refiner"] == { - "root": model_dir / "refiner", - "gemma_root": model_dir / "refiner/text_encoder", - "seed": 42, - } - assert captured["generation"] == { - "num_frames": 321, - "fps": 16, - "step": 60, - "cfg_scale": 5.0, - "flow_shift": 9.8, - "seed": 42, - } - assert len(captured["generate"]) == 3 - assert len(synchronize_calls) == 5 - result = json.loads(output.read_text(encoding="utf-8")) - assert result["output_summary"]["num_frames"] == 321 - - -def test_sana_reference_requires_action_from_current_request(tmp_path: Path) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json" - ) - with pytest.raises(ValueError, match="non-empty action"): - task_reference._run_sana_wm( - arguments, - {"prompt": "drive", "image_path": "assets/demo_0.png"}, - {"reference_repo": str(checkout)}, - ) - - -@pytest.mark.parametrize("frame_controls", [{"num_frames": 321}, {}]) -def test_sana_task_reference_uses_one_explicit_official_command( - monkeypatch, tmp_path: Path, frame_controls: dict, -) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - model_dir = tmp_path / "model" - model_dir.mkdir() - captured = {} - - def run(command, **kwargs): - captured["command"] = command - captured["kwargs"] = kwargs - output = Path(command[command.index("--output") + 1]) - output.write_text( - json.dumps( - { - "samples_ms": [1.0, 2.0], - "output_summary": { - "media_type": "video", - "media_count": 321, - "num_frames": 321, - "height": 704, - "width": 1280, - "channels": 3, - }, - } - ), - encoding="utf-8", - ) - return SimpleNamespace(returncode=0, stderr="") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", - warmup=1, - iterations=2, - ) - result = task_reference._run_sana_wm( - arguments, - { - "prompt": "drive forward", - "image_path": "assets/demo_0.png", - "action": "w-80,jw-40,w-40,lw-60,w-100", - "translation_speed": 0.055, - "rotation_speed_deg": 1.2, - **frame_controls, - "fps": 16, - "num_steps": 60, - "cfg_scale": 5.0, - "flow_shift": 9.8, - "seed": 42, - "no_action_overlay": True, - }, - { - "reference_repo": str(checkout), - "model_dir": str(model_dir), - "intrinsics": "assets/demo_0_intrinsics.npy", - }, - ) - - command = captured["command"] - assert command[command.index("--reference-repo") + 1] == str(checkout.resolve()) - assert command[command.index("--translation_speed") + 1] == "0.055" - assert command[command.index("--rotation_speed_deg") + 1] == "1.2" - assert command[command.index("--num_frames") + 1] == "321" - assert command[command.index("--fps") + 1] == "16" - assert command[command.index("--flow_shift") + 1] == "9.8" - assert command[command.index("--refiner_seed") + 1] == "42" - assert command[command.index("--warmup") + 1] == "1" - assert command[command.index("--iterations") + 1] == "2" - assert "--no_action_overlay" in command - assert "env" not in captured["kwargs"] - assert result[0] == [1.0, 2.0] - assert result[1]["num_frames"] == 321 - - -@pytest.mark.parametrize("controls,manifest_frames,expected", [ - ({}, 321, "321"), - ({"num_frames": 7}, 321, "7"), - ({"num_frames": 0}, 321, "0"), - ({"config": {"num_frames": 0}}, 321, "0"), - ({"num_frames": 7}, None, "7"), - ({}, None, None), -]) -def test_sana_semantic_reference_uses_manifest_frames_only_when_absent( - monkeypatch, tmp_path: Path, controls: dict, manifest_frames: int | None, expected: str | None, -) -> None: - model = perf.ManifestCatalog(REPO / "families").resolve("sana-wm-bidirectional") - model = replace(model, testcases=({**model.testcases[0], **controls},)) - case = perf.resolve_case(model, tmp_path / "model.bundle", selected_task="image_text_action_to_video") - request = task_reference.flatten_config(case.request) - before = dict(request) - manifest = json.loads(model.manifest_path.read_text()) - if manifest_frames is None: - del manifest["video_num_frames"] - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text(json.dumps(manifest)) - checkout = tmp_path / "Sana" - checkout.mkdir() - options = { - "reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), - "intrinsics": str(model.manifest_path.parent.parent / "assets/demo_0_intrinsics.npy"), - } - testcase = next(value for value in manifest["testcases"] if value["name"] == case.testcase_name) - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay"): - options[name] = testcase[name] - assert name not in request - - def run(command, **kwargs): - assert command[command.index("--num_frames") + 1] == expected - assert command[command.index("--action") + 1] == request["action"] - assert command[command.index("--translation_speed") + 1] == "0.055" - assert command[command.index("--rotation_speed_deg") + 1] == "1.2" - assert command[command.index("--fps") + 1] == "16" - assert command[command.index("--flow_shift") + 1] == "9.8" - assert "--no_action_overlay" in command - raise RuntimeError("captured reference command") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - arguments = SimpleNamespace(manifest=manifest_path, warmup=1, iterations=2) - if expected is None: - with pytest.raises(KeyError, match="video_num_frames"): - task_reference._run_sana_wm(arguments, request, options) - else: - with pytest.raises(RuntimeError, match="captured reference command"): - task_reference._run_sana_wm(arguments, request, options) - assert request == before - - -@pytest.mark.parametrize("source", ["request", "config", "options"]) -@pytest.mark.parametrize("controls", [ - {"translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True}, - {"translation_speed": 0.0, "rotation_speed_deg": 0.0, - "fps": 0, "flow_shift": 0.0, "no_action_overlay": False}, -]) -def test_sana_reference_control_presence_and_request_precedence( - monkeypatch, tmp_path: Path, source: str, controls: dict, -) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", - warmup=1, iterations=2, - ) - request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", - "action": "w-80,jw-40,w-40,lw-60,w-100", "num_frames": 321, - "num_steps": 60, "cfg_scale": 5.0, "seed": 42} - options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), - "translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - if source == "options": - options.update(controls) - elif source == "config": - request = task_reference.flatten_config({**request, "config": controls}) - else: - request.update(controls) - before = dict(request), dict(options) - - def run(command, **kwargs): - for name in ("translation_speed", "rotation_speed_deg", "fps", "flow_shift"): - assert command[command.index("--" + name) + 1] == str(controls[name]) - assert ("--no_action_overlay" in command) is controls["no_action_overlay"] - assert command[command.index("--action") + 1] == request["action"] - assert command[command.index("--refiner_seed") + 1] == "42" - assert "env" not in kwargs - raise RuntimeError("captured reference controls") - - monkeypatch.setattr(task_reference.subprocess, "run", run) - with pytest.raises(RuntimeError, match="captured reference controls"): - task_reference._run_sana_wm(arguments, request, options) - assert (request, options) == before - - -@pytest.mark.parametrize("missing", [ - "translation_speed", "rotation_speed_deg", "fps", "flow_shift", "no_action_overlay", -]) -def test_sana_reference_control_missing_from_request_and_options_still_errors( - monkeypatch, tmp_path: Path, missing: str, -) -> None: - checkout = tmp_path / "Sana" - checkout.mkdir() - arguments = SimpleNamespace( - manifest=REPO / "families/sana_wm/tests/manifests/sana-wm-bidirectional.json", - warmup=1, iterations=2, - ) - request = {"prompt": "drive forward", "image_path": "assets/demo_0.png", "action": "w-320", - "num_frames": 321, "num_steps": 60, "cfg_scale": 5.0, "seed": 42} - options = {"reference_repo": str(checkout), "model_dir": str(tmp_path / "model"), - "translation_speed": 0.055, "rotation_speed_deg": 1.2, - "fps": 16, "flow_shift": 9.8, "no_action_overlay": True} - del options[missing] - monkeypatch.setattr(task_reference.subprocess, "run", lambda *args, **kwargs: pytest.fail("missing control must fail before reference execution")) - with pytest.raises(KeyError, match=missing): - task_reference._run_sana_wm(arguments, request, options) - def test_output_contracts_are_closed_and_semantic(tmp_path: Path) -> None: _, environment = _environment(tmp_path) @@ -3033,3 +1955,122 @@ def test_output_contracts_are_closed_and_semantic(tmp_path: Path) -> None: ) with pytest.raises(perf.PerfMatrixError, match="unsupported output contract"): perf._contract_name(bad) + + +def test_detection_output_contract_matches_by_class_and_iou() -> None: + entry = SimpleNamespace( + spec={ + "baseline": { + "output_contract": "detection-parity", + "min_box_iou": 0.9, + "max_score_abs_error": 0.05, + } + } + ) + candidate = { + "output_summary": { + "boxes": [10.0, 10.0, 50.0, 50.0], + "scores": [0.91], + "class_ids": [7], + } + } + reference = { + "output_summary": { + "boxes": [[10.5, 10.0, 50.0, 50.0]], + "scores": [0.9], + "class_ids": [7], + } + } + + matched, reason, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert reason == "" + assert evidence["minimum_box_iou"] >= 0.9 + reference["output_summary"]["class_ids"] = [8] + assert perf._output_contract(entry, candidate, reference)[0] is False + + +def test_prompted_mask_contract_matches_binary_semantics() -> None: + entry = SimpleNamespace( + spec={"baseline": {"output_contract": "prompted-mask-parity", "min_mask_iou": 0.7}} + ) + candidate = { + "output_summary": { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "logits", + "masks": [2.0, -1.0, -1.0, 3.0], + } + } + reference = { + "output_summary": { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [1, 0, 0, 1], + } + } + + matched, reason, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert reason == "" + assert evidence == {"masks": 1, "minimum_mask_iou": 1.0, "required_mask_iou": 0.7} + reference["output_summary"]["masks"] = [0, 1, 1, 0] + assert perf._output_contract(entry, candidate, reference)[0] is False + + +def test_instance_mask_contract_checks_masks_boxes_and_scores() -> None: + entry = SimpleNamespace( + spec={ + "baseline": { + "output_contract": "instance-mask-parity", + "min_mask_iou": 0.7, + "min_box_iou": 0.9, + "max_score_abs_error": 0.05, + } + } + ) + value = { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [1, 0, 0, 1], + "iou_scores": [0.9], + "boxes": [[0.0, 0.0, 2.0, 2.0]], + "box_coordinates": "original_image_pixels_xyxy", + } + candidate = {"output_summary": value} + reference = {"output_summary": dict(value)} + + matched, reason, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert reason == "" + assert evidence["minimum_mask_iou"] == evidence["minimum_box_iou"] == 1.0 + reference["output_summary"]["iou_scores"] = [0.7] + assert perf._output_contract(entry, candidate, reference)[0] is False + + +def test_vision_language_text_contract_allows_small_normalized_differences() -> None: + entry = SimpleNamespace( + spec={ + "baseline": { + "output_contract": "vision-language-text", + "max_normalized_edit_distance": 0.15, + } + } + ) + candidate = {"output_summary": {"text": "Red."}} + reference = {"output_summary": {"text": "red"}} + + matched, _, evidence = perf._output_contract(entry, candidate, reference) + + assert matched is True + assert evidence["normalized_edit_distance"] <= 0.15 + candidate["output_summary"]["text"] = "blue" + assert perf._output_contract(entry, candidate, reference)[0] is False diff --git a/qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py b/qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py new file mode 100644 index 0000000000..c63c67c045 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path + +from qualification_tests.benchmark_qualification.performance import reference_protocol + + +def _command(tmp_path: Path, *, script: Path | None, adapter: str | None) -> list[str]: + return reference_protocol.command( + python=Path("/reference/bin/python"), + generic_runner=Path("/repo/generic_reference.py"), + script=script, + family="example", + operation="generate", + manifest=tmp_path / "model.json", + selected_task="text_generation" if script else None, + testcase_name="smoke", + adapter=adapter, + adapter_options={"option": 1}, + timing_contract={"timing_scope": "task-model-call-wall"}, + padding="longest", + model="owner/model", + revision="a" * 40, + request={"prompt": "hello"}, + precision="fp32", + mode="hf-eager", + warmup=0, + iterations=1, + case_name="example", + output=tmp_path / "reference.json", + trust_remote_code=False, + local_files_only=True, + ) + + +def test_generic_reference_command_has_no_family_dispatch(tmp_path: Path) -> None: + command = _command(tmp_path, script=None, adapter="hf-transformers-vision") + + assert command[:2] == ["/reference/bin/python", "/repo/generic_reference.py"] + assert command[command.index("--adapter") + 1] == "hf-transformers-vision" + assert "--family" not in command + assert command[command.index("--testcase-name") + 1] == "smoke" + assert json.loads(command[command.index("--request-json") + 1]) == {"prompt": "hello"} + + +def test_family_reference_command_names_only_its_owner(tmp_path: Path) -> None: + script = tmp_path / "families/example/tests/benchmark/reference.py" + command = _command(tmp_path, script=script, adapter=None) + + assert command[:2] == ["/reference/bin/python", str(script)] + assert command[command.index("--family") + 1] == "example" + assert command[command.index("--selected-task") + 1] == "text_generation" + assert "--adapter" not in command + assert "--testcase-name" not in command + + +def test_reference_command_rejects_ambiguous_implementation(tmp_path: Path) -> None: + try: + _command(tmp_path, script=None, adapter=None) + except ValueError as error: + assert "exactly one" in str(error) + else: + raise AssertionError("ambiguous reference must fail") diff --git a/qualification_tests/benchmark_qualification/performance/tests/test_reporting.py b/qualification_tests/benchmark_qualification/performance/tests/test_reporting.py new file mode 100644 index 0000000000..d4c655b620 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/tests/test_reporting.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest + +from qualification_tests.benchmark_qualification.performance import matrix as perf_matrix + + +@pytest.mark.parametrize( + "schema, row_key, reference_key", + [("v1", "cases", "baseline"), ("v2", "rows", "reference")], +) +def test_performance_report_preserves_historical_measurements( + tmp_path, schema, row_key, reference_key +): + scope = "model_call_wall" if schema == "v1" else "public_task_call_wall" + results = { + "schema_version": "trtmc.perf-matrix/" + schema, + "status": "completed", + "selected_entry_ids": ["example.generate"], + row_key: [ + { + "id": "example.generate", + "model": "example", + "operation": "generate", + "status": "green", + "candidate": {"samples_ms": [8.0, 10.0, 12.0], "timing_scope": scope}, + reference_key: { + "samples_ms": [12.0, 14.0, 16.0], + "measurement_policy": {"timing_scope": "public_operation_call_wall"}, + }, + "commands": { + "candidate": {"argv": ["trtmc-bench", "run", "--model", "example"]} + }, + } + ], + } + original = json.dumps(results) + (tmp_path / "results.json").write_text(original) + assert perf_matrix.main(["report", str(tmp_path)]) == 0 + report = json.loads((tmp_path / "report.json").read_text()) + document = (tmp_path / "report.html").read_text() + assert report["source_schema_version"] == results["schema_version"] + assert report["rows"][0][reference_key] == results[row_key][0][reference_key] + assert "p50: 10.000 ms" in document + assert "p95: 11.800 ms" in document + assert scope in document + assert "public_operation_call_wall" in document + assert "trtmc-bench" in document + assert 'id="filter"' in document + assert (tmp_path / "results.json").read_text() == original + if schema == "v1": + with pytest.raises(perf_matrix.PerfMatrixError, match="unsupported results schema"): + perf_matrix._load_results(tmp_path) diff --git a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py b/qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py similarity index 83% rename from apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py rename to qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py index 6428902993..f730c71d05 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_structured_output_contracts.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py @@ -10,7 +10,7 @@ import pytest -import tools.perf_matrix as perf +from qualification_tests.benchmark_qualification.performance import matrix as perf OPERATIONS = { @@ -39,6 +39,25 @@ def _entry(operation): ) +def _control_entry(): + return perf.ResolvedEntry( + spec={"id": "robot-action-fixture", "operation": "control", "baseline": {}}, + model=SimpleNamespace(task="robot_control"), + case=SimpleNamespace( + selected_task=None, + request={}, + measurement=SimpleNamespace(asset_loading_included=False), + ), + manifest={}, + reference_precision="fp32", + baseline_timing={ + "timing_scope": "task-pipeline-call-wall", + "input_preparation_included": True, + "asset_loading_included": False, + }, + ) + + def _geometry(root: Path): root.mkdir() # Two HWC pixels: a valid metric point and an invalid +Inf sentinel. @@ -109,7 +128,22 @@ def _poses(_root: Path): } -FIXTURES = {"geometry": _geometry, "predict_structure": _structure, "refine_pose": _poses} +def _actions(_root: Path): + return { + "action_steps": 2, + "action_dim": 2, + "action_values": 4, + "actions": [0.1, -0.2, 0.3, -0.4], + "finite": True, + "within_training_bounds": True, + } + + +FIXTURES = { + "geometry": _geometry, + "predict_structure": _structure, + "refine_pose": _poses, +} def _compare(operation, left, right): @@ -283,6 +317,78 @@ def test_geometry_shape_does_not_filter_invalid_infinity_or_add_depth_accuracy_r assert struct.unpack("<2f", originals["depth_artifact"])[1] == float("inf") +def test_geometry_parity_compares_complete_numerical_artifacts(tmp_path): + left = _geometry(tmp_path / "candidate") + right = _geometry(tmp_path / "reference") + entry = _entry("geometry") + entry.spec["baseline"].update( + output_contract="metric-geometry-parity", + mask_iou=0.999, + depth_absrel_mean=0.005, + depth_rel_l2=0.02, + points_rel_l2=0.02, + points_cosine=0.99999, + intrinsics_max_relative_error=0.002, + point_depth_consistency=0.00001, + ) + + matched, reason, evidence = perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + ) + + assert matched is True and reason == "" + assert evidence["numerical_parity_checked"] is True + Path(right["depth_artifact"]).write_bytes(struct.pack("<2f", 6, float("inf"))) + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + + +def test_robot_action_parity_compares_every_action_value(tmp_path): + left = _actions(tmp_path / "candidate") + right = _actions(tmp_path / "reference") + entry = _control_entry() + entry.spec["baseline"].update( + output_contract="robot-action-parity", + action_max_abs_error=0.00005, + action_mean_abs_error=0.000005, + action_rmse=0.00001, + ) + + matched, reason, evidence = perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + ) + + assert matched is True and reason == "" + assert evidence["numerical_parity_checked"] is True + right["actions"][2] += 0.01 + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + + +def test_robot_action_parity_rejects_missing_or_out_of_bounds_values(tmp_path): + left = _actions(tmp_path / "candidate") + right = _actions(tmp_path / "reference") + entry = _control_entry() + entry.spec["baseline"].update( + output_contract="robot-action-parity", + action_max_abs_error=0.00005, + action_mean_abs_error=0.000005, + action_rmse=0.00001, + ) + + left["within_training_bounds"] = False + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + left["within_training_bounds"] = True + right["actions"].pop() + assert perf._output_contract( + entry, {"output_summary": left}, {"output_summary": right} + )[0] is False + + def test_optional_reference_calibration_artifact_is_not_silently_ignored(tmp_path): left = _geometry(tmp_path / "candidate") right = _geometry(tmp_path / "reference") diff --git a/apps/benchmark/trtmc_benchmark/tests/test_timing_contracts.py b/qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py similarity index 81% rename from apps/benchmark/trtmc_benchmark/tests/test_timing_contracts.py rename to qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py index 1bab6383d6..968b4629e3 100644 --- a/apps/benchmark/trtmc_benchmark/tests/test_timing_contracts.py +++ b/qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py @@ -11,9 +11,9 @@ import pytest -import tools.perf_matrix as perf -from apps.benchmark.performance.baselines import task_reference -from apps.benchmark.performance.baselines.timing_contracts import timing_contract +from qualification_tests.benchmark_qualification.performance import matrix as perf +from qualification_tests.benchmark_qualification.performance.references import generic_reference +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract from trtmc_benchmark.types import ModelDescriptor @@ -23,7 +23,7 @@ def _declaration(scope="task-pipeline-call-wall", preparation=True, assets=False def test_every_release_entry_preserves_its_existing_explicit_timing(): - _, entries, _ = perf.load_suite(perf.REPOSITORY / "apps/benchmark/performance/release.yaml") + _, entries, _ = perf.load_suite(perf.REPOSITORY / "qualification_tests/benchmark_qualification/performance/config/release.yaml") assert entries for entry in entries: baseline = entry["baseline"] @@ -102,7 +102,7 @@ def invoke(): calls.append("invoke") return {"value": 7} - return task_reference.Session(invoke, "fixture", **actual) + return generic_reference.Session(invoke, "fixture", **actual) def measure(session, warmup, iterations): calls.append("measure") @@ -110,9 +110,9 @@ def measure(session, warmup, iterations): summary = session.invoke() return [2.0] * iterations, summary - monkeypatch.setitem(task_reference.LOADERS, "hf-transformers-vision", load) - monkeypatch.setattr(task_reference, "_measure", measure) - monkeypatch.setattr(task_reference, "_environment", lambda: {}) + monkeypatch.setitem(generic_reference.LOADERS, "hf-transformers-vision", load) + monkeypatch.setattr(generic_reference, "_measure", measure) + monkeypatch.setattr(generic_reference, "_environment", lambda: {}) return calls @@ -121,7 +121,7 @@ def test_real_session_metadata_not_family_name_controls_validation(tmp_path, mon actual = _declaration("task-model-call-wall", False) calls = _loader(monkeypatch, actual) arguments = _arguments(tmp_path, actual, family) - assert task_reference.run(arguments) == 0 + assert generic_reference.run(arguments) == 0 result = json.loads(arguments.output.read_text(encoding="utf-8")) assert {name: result["measurement_policy"][name] for name in actual} == actual assert calls == ["load", "measure", "invoke", "invoke", "invoke"] @@ -135,7 +135,7 @@ def test_session_timing_drift_is_rejected_before_warmup_or_measurement(tmp_path, calls = _loader(monkeypatch, actual) arguments = _arguments(tmp_path, _declaration()) with pytest.raises(RuntimeError, match="timing drifted"): - task_reference.run(arguments) + generic_reference.run(arguments) assert calls == ["load"] assert not arguments.output.exists() @@ -144,26 +144,15 @@ def test_unconfigured_fixed_loader_reports_its_actual_session_boundary(tmp_path, actual = _declaration("task-model-call-wall", False) _loader(monkeypatch, actual) arguments = _arguments(tmp_path, {}) - assert task_reference.run(arguments) == 0 + assert generic_reference.run(arguments) == 0 result = json.loads(arguments.output.read_text(encoding="utf-8")) assert {name: result[name] for name in actual} == actual def test_ambiguous_embedding_reference_requires_existing_explicit_timing_option(): with pytest.raises(ValueError, match="requires explicit --timing-contract-json"): - task_reference._load_embedding(SimpleNamespace(family="bert"), {"prompt": "hello"}, {}) + generic_reference._load_embedding(SimpleNamespace(family="bert"), {"prompt": "hello"}, {}) with pytest.raises(ValueError, match="preloads its assets"): - task_reference._load_embedding( + generic_reference._load_embedding( SimpleNamespace(timing_contract_json=json.dumps(_declaration(assets=True))), {}, {}, ) - - -def test_delegated_runner_cannot_echo_the_expected_asset_flag(tmp_path, monkeypatch): - arguments = _arguments(tmp_path, _declaration(assets=True)) - arguments.adapter, arguments.mode = "upstream-lance", "pytorch-eager" - monkeypatch.setattr(task_reference, "_run_lance", lambda *_: ( - [2.0, 3.0], {}, "fixture", "task-pipeline-call-wall", True, False, - )) - with pytest.raises(RuntimeError, match="timing drifted"): - task_reference.run(arguments) - assert not arguments.output.exists() diff --git a/qualification_tests/benchmark_qualification/performance/types.py b/qualification_tests/benchmark_qualification/performance/types.py new file mode 100644 index 0000000000..f7c3172b22 --- /dev/null +++ b/qualification_tests/benchmark_qualification/performance/types.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stable values exchanged by Performance Qualification modules.""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + + +class PerfMatrixError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Environment: + name: str + trtmc_bench: Path + worker: Path + hf_runner: Path + task_runner: Path + results_root: Path + scratch_root: Path + bundle_cache: Path + bundle_roots: tuple[Path, ...] + runtime_root: Path + bundle_retention: str + local_files_only: bool + timeout_seconds: int + references: Mapping[str, str] + reference_python: Path = Path(sys.executable) + storage_root: Path | None = None + hf_cache_mode: str = "shared" + hf_cache_retention: str = "retain" + + +@dataclass(frozen=True) +class ResolvedEntry: + spec: Mapping[str, Any] + model: Any + case: Any + manifest: Mapping[str, Any] + reference_precision: str + baseline_timing: Mapping[str, Any] diff --git a/qualification_tests/benchmark_qualification/references/hf_encoder.py b/qualification_tests/benchmark_qualification/references/hf_encoder.py new file mode 100644 index 0000000000..7f381437a3 --- /dev/null +++ b/qualification_tests/benchmark_qualification/references/hf_encoder.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Produce Hugging Face encoder vectors for STS qualification.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any, Mapping + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + import transformers + + if not torch.cuda.is_available(): + raise RuntimeError("the HF encoder Accuracy reference requires CUDA") + precision = str(request.get("precision", "fp32")) + dtypes = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16} + if precision not in dtypes: + raise ValueError(f"unsupported reference precision {precision!r}") + mode = _vector_mode(str(request.get("mode", "cls"))) + model_class, tokenizer_class = _reference_classes( + transformers, + str(request.get("model_class", "auto")), + str(request.get("tokenizer_class", "auto")), + ) + load_options = _model_load_options(request) + tokenizer_options = { + name: load_options[name] + for name in ("local_files_only", "revision", "trust_remote_code") + if name in load_options + } + tokenizer = tokenizer_class.from_pretrained(str(request["model"]), **tokenizer_options) + model = model_class.from_pretrained( + str(request["model"]), torch_dtype=dtypes[precision], **load_options + ).eval() + model.to("cuda") + max_length = int(request.get("max_length", 512)) + if max_length < 1: + raise ValueError("max_length must be positive") + + results = [] + with torch.inference_mode(): + for sample in request["samples"]: + encoded = tokenizer( + str(sample["prompt"]), + return_tensors="pt", + truncation=True, + max_length=max_length, + ) + encoded = {name: value.to("cuda") for name, value in encoded.items()} + outputs = model(**encoded, output_hidden_states=True) + hidden = getattr(outputs, "last_hidden_state", None) + if hidden is None: + hidden_states = getattr(outputs, "hidden_states", None) + if hidden_states: + hidden = hidden_states[-1] + elif isinstance(outputs, (tuple, list)) and outputs: + hidden = outputs[0] + if hidden is None or hidden.ndim != 3: + raise RuntimeError( + f"HF encoder output for {sample['sample_id']} has no rank-3 hidden state" + ) + if mode == "embedding": + attention_mask = encoded.get("attention_mask") + if attention_mask is None: + attention_mask = torch.ones(hidden.shape[:2], device=hidden.device) + mask = attention_mask.unsqueeze(-1).to(hidden.dtype) + vector = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1) + vector = torch.nn.functional.normalize(vector, p=2, dim=-1)[0] + else: + vector = hidden[0, 0] + results.append( + { + "sample_id": str(sample["sample_id"]), + "pair_id": str(sample["pair_id"]), + "pair_side": str(sample["pair_side"]), + "score": float(sample["score"]), + "vector": [float(value) for value in vector.float().cpu().tolist()], + } + ) + output = { + "schema_version": "trtmc.encoder-reference/v1", + "model": request["model"], + "revision": getattr(model.config, "_commit_hash", None) or request.get("revision"), + "precision": precision, + "mode": mode, + "samples": results, + } + arguments.output.write_text( + json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +def _vector_mode(value: str) -> str: + if value not in {"cls", "embedding"}: + raise ValueError(f"unsupported encoder vector mode {value!r}") + return value + + +def _reference_classes( + transformers_module: Any, model: str, tokenizer: str +) -> tuple[Any, Any]: + return ( + _transformers_class(transformers_module, model, "AutoModel"), + _transformers_class(transformers_module, tokenizer, "AutoTokenizer"), + ) + + +def _transformers_class(module: Any, value: str, automatic: str) -> Any: + name = automatic if value == "auto" else value.removeprefix("transformers.") + if value != "auto" and (not value.startswith("transformers.") or not name.isidentifier()): + raise ValueError(f"unsupported Transformers class {value!r}") + resolved = getattr(module, name, None) + if resolved is None: + raise ValueError(f"unsupported Transformers class {value!r}") + return resolved + + +def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: + options: dict[str, Any] = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + revision = request.get("revision") + if revision: + options["revision"] = revision + trust_remote_code = request.get("trust_remote_code") + if trust_remote_code is not None: + if not isinstance(trust_remote_code, bool): + raise ValueError("trust_remote_code must be a boolean") + options["trust_remote_code"] = trust_remote_code + return options + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/qualification_tests/benchmark_qualification/references/hf_text_generation.py b/qualification_tests/benchmark_qualification/references/hf_text_generation.py new file mode 100644 index 0000000000..c8dd869949 --- /dev/null +++ b/qualification_tests/benchmark_qualification/references/hf_text_generation.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Produce deterministic Hugging Face text-generation qualification references.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any, Mapping, Sequence + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + request = json.loads(arguments.request.read_text(encoding="utf-8")) + + import torch + from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer + + if not torch.cuda.is_available(): + raise RuntimeError("the HF Accuracy reference requires CUDA") + precision = str(request.get("precision", "fp32")) + dtypes = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16} + if precision not in dtypes: + raise ValueError(f"unsupported reference precision {precision!r}") + model_options = _model_load_options(request) + tokenizer_options = { + name: model_options[name] + for name in ("local_files_only", "revision", "trust_remote_code") + if name in model_options + } + tokenizer = AutoTokenizer.from_pretrained(str(request["model"]), **tokenizer_options) + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + task = str(request.get("task", "causal-lm")) + model_type = _model_class(task, AutoModelForCausalLM, AutoModelForSeq2SeqLM) + model = model_type.from_pretrained( + str(request["model"]), **_precision_load_options(dtypes[precision]), **model_options + ).eval() + model.to("cuda") + generation = request.get("generation", {}) + if not isinstance(generation, Mapping): + raise ValueError("generation must be an object") + translation, source_language_token_id = _translation_controls(tokenizer, generation) + results = [] + with torch.inference_mode(): + for sample in request["samples"]: + prompt = _truncate( + tokenizer, + str(sample["prompt"]), + int(request["prompt_token_limit"]), + str(request.get("truncation_side", "left")), + ) + encoded = tokenizer(prompt, return_tensors="pt") + _apply_source_language(encoded, source_language_token_id, tokenizer) + encoded = {name: value.to("cuda") for name, value in encoded.items()} + options: dict[str, Any] = { + "max_new_tokens": int(generation.get("max_new_tokens", 64)), + "do_sample": bool(generation.get("do_sample", False)), + "num_beams": 1, + "pad_token_id": tokenizer.eos_token_id, + "eos_token_id": tokenizer.eos_token_id, + "repetition_penalty": float(generation.get("repetition_penalty", 1.0)), + **translation, + } + if options["do_sample"]: + options.update( + temperature=float(generation.get("temperature", 1.0)), + top_k=int(generation.get("top_k", 0)), + top_p=float(generation.get("top_p", 1.0)), + ) + seed = int(generation.get("seed", -1)) + if seed >= 0: + torch.manual_seed(seed) + generated = model.generate(**encoded, **options) + prompt_length = 0 if task == "seq2seq-lm" else int(encoded["input_ids"].shape[-1]) + token_ids = generated[0, prompt_length:].detach().cpu().tolist() + if task == "seq2seq-lm": + generation_config = getattr(model, "generation_config", model.config) + token_ids = _normalize_seq2seq_tokens( + token_ids, + getattr(generation_config, "decoder_start_token_id", None), + getattr(generation_config, "eos_token_id", None), + str(request.get("output_token_policy", "new-tokens")), + ) + results.append( + { + "sample_id": str(sample["sample_id"]), + "prompt": prompt, + "token_ids": [int(value) for value in token_ids], + "text": tokenizer.decode(token_ids, skip_special_tokens=True), + } + ) + output = { + "schema_version": "trtmc.accuracy-reference/v1", + "model": request["model"], + "revision": getattr(model.config, "_commit_hash", None) or request.get("revision"), + "task": task, + "precision": precision, + "output_token_policy": str(request.get("output_token_policy", "new-tokens")), + "samples": results, + } + arguments.output.write_text( + json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +def _model_load_options(request: Mapping[str, Any]) -> dict[str, Any]: + options: dict[str, Any] = { + "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" + } + if request.get("revision"): + options["revision"] = request["revision"] + trust_remote_code = request.get("trust_remote_code") + if trust_remote_code is not None: + if not isinstance(trust_remote_code, bool): + raise ValueError("trust_remote_code must be a boolean") + options["trust_remote_code"] = trust_remote_code + experts_implementation = request.get("experts_implementation") + if experts_implementation is not None: + if not isinstance(experts_implementation, str) or not experts_implementation: + raise ValueError("experts_implementation must be a non-empty string") + options["experts_implementation"] = experts_implementation + return options + + +def _model_class(task: str, causal: Any, seq2seq: Any) -> Any: + if task == "causal-lm": + return causal + if task == "seq2seq-lm": + return seq2seq + raise ValueError(f"unsupported reference task {task!r}") + + +def _translation_controls( + tokenizer: Any, request: Mapping[str, Any] +) -> tuple[dict[str, int], int | None]: + source = request.get("source_language") + target = request.get("target_language") + source_placement = request.get("source_language_placement") + for name, value in (("source_language", source), ("target_language", target)): + if value is not None and (not isinstance(value, str) or not value): + raise ValueError(f"generation.{name} must be a nonempty language identifier") + if source_placement is not None and source_placement != "replace-final-unk": + raise ValueError( + "generation.source_language_placement must be replace-final-unk when set" + ) + + def token_id(language: str) -> int: + lookup = getattr(tokenizer, "get_lang_id", None) + if lookup is None: + lookup = tokenizer.convert_tokens_to_ids + value = lookup(language) + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value == getattr(tokenizer, "unk_token_id", None) + ): + raise ValueError(f"tokenizer does not recognize language {language!r}") + return value + + def explicit_id(name: str) -> int | None: + value = request.get(name) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value < -1: + raise ValueError(f"generation.{name} must be a nonnegative integer or -1") + return None if value == -1 else value + + source_id = explicit_id("source_language_token_id") + target_id = explicit_id("forced_bos_token_id") + if source_placement is not None and source_id is None: + raise ValueError( + "generation.source_language_placement requires source_language_token_id" + ) + manual_source_id = None + if source_id is not None: + if source is not None and token_id(source) != source_id: + raise ValueError("source language disagrees with source_language_token_id") + source = tokenizer.convert_ids_to_tokens(source_id) + if source is not None: + if hasattr(tokenizer, "src_lang"): + token_id(source) + tokenizer.src_lang = source + elif source == getattr(tokenizer, "source_lang", None): + pass + elif source_id is not None and token_id(source) == source_id: + if source_placement != "replace-final-unk": + raise ValueError( + "generic translation tokenizer requires explicit " + "source_language_placement=replace-final-unk" + ) + manual_source_id = source_id + else: + raise ValueError("reference tokenizer does not support the requested source language") + if target is not None: + if hasattr(tokenizer, "src_lang"): + resolved = token_id(target) + if target_id is not None and target_id != resolved: + raise ValueError("target language disagrees with forced_bos_token_id") + target_id = resolved + elif target == getattr(tokenizer, "target_lang", None): + pass + elif target_id is not None and token_id(target) == target_id: + pass + else: + raise ValueError("reference tokenizer does not support the requested target language") + controls = {} if target_id is None else {"forced_bos_token_id": target_id} + return controls, manual_source_id + + +def _apply_source_language( + encoded: Mapping[str, Any], source_token_id: int | None, tokenizer: Any +) -> None: + if source_token_id is None: + return + input_ids = encoded.get("input_ids") + if input_ids is None or getattr(input_ids, "ndim", None) != 2: + raise ValueError("tokenized translation input must contain rank-2 input_ids") + attention_mask = encoded.get("attention_mask") + for row in range(int(input_ids.shape[0])): + if attention_mask is None: + index = int(input_ids.shape[1]) - 1 + else: + positions = attention_mask[row].nonzero(as_tuple=False) + if int(positions.numel()) == 0: + raise ValueError("tokenized translation input cannot be empty") + index = int(positions[-1].item()) + current = int(input_ids[row, index].item()) + if current == source_token_id: + continue + if current != getattr(tokenizer, "unk_token_id", None): + raise ValueError("generic translation tokenizer did not emit a language placeholder") + input_ids[row, index] = source_token_id + + +def _normalize_seq2seq_tokens( + token_ids: Sequence[int], + decoder_start_token_id: int | None, + eos_token_id: int | Sequence[int] | None, + policy: str, +) -> list[int]: + if policy not in {"new-tokens", "strip-start", "strip-start-and-eos"}: + raise ValueError(f"unsupported output token policy {policy!r}") + normalized = [int(value) for value in token_ids] + if policy == "new-tokens": + return normalized + if ( + normalized + and decoder_start_token_id is not None + and normalized[0] == int(decoder_start_token_id) + ): + normalized.pop(0) + if policy != "strip-start-and-eos" or not normalized or eos_token_id is None: + return normalized + eos_ids = ( + {int(eos_token_id)} + if isinstance(eos_token_id, int) + else {int(value) for value in eos_token_id} + ) + if normalized[-1] in eos_ids: + normalized.pop() + return normalized + + +def _precision_load_options(dtype: Any) -> dict[str, Any]: + return {"torch_dtype": dtype} + + +def _truncate(tokenizer: Any, prompt: str, limit: int, side: str) -> str: + if limit < 1 or side not in {"left", "right"}: + raise ValueError("prompt token limit and truncation side are invalid") + token_ids = [int(value) for value in tokenizer.encode(prompt, add_special_tokens=False)] + if len(token_ids) <= limit: + return prompt + selected = token_ids[-limit:] if side == "left" else token_ids[:limit] + return str( + tokenizer.decode( + selected, + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/qualification_tests/benchmark_qualification/references/speech_io.py b/qualification_tests/benchmark_qualification/references/speech_io.py new file mode 100644 index 0000000000..b662f4d9d1 --- /dev/null +++ b/qualification_tests/benchmark_qualification/references/speech_io.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic request and PCM mechanics for family-owned ASR references.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import math +from pathlib import Path +from typing import Any, Mapping, Sequence + + +@dataclass(frozen=True) +class SpeechSample: + sample_id: str + gold_text: str + waveform: Any + sample_rate: int + wav_path: Path + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser() + value.add_argument("--request", required=True, type=Path) + value.add_argument("--output", required=True, type=Path) + return value + + +def load_request(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("speech reference request must be an object") + return value + + +def prepare_samples(request: Mapping[str, Any], output: Path) -> list[SpeechSample]: + import numpy as np + from scipy.signal import resample_poly + import soundfile as sf + + configured = request.get("samples") + if not isinstance(configured, list) or not configured: + raise ValueError("speech reference requires a non-empty samples list") + audio_root = output.parent / "reference-audio" + audio_root.mkdir(parents=True, exist_ok=True) + prepared = [] + for index, value in enumerate(configured): + if not isinstance(value, Mapping): + raise ValueError(f"speech reference sample {index} must be an object") + sample_id = required_string(value.get("sample_id"), "sample_id") + gold_text = required_string(value.get("gold_text"), "gold_text") + source = Path(required_string(value.get("audio_path"), "audio_path")) + if not source.is_file(): + raise FileNotFoundError(source) + waveform, sample_rate = sf.read(source, dtype="float32", always_2d=True) + waveform = waveform.mean(axis=1) + target_rate = 16_000 + if int(sample_rate) != target_rate: + divisor = math.gcd(int(sample_rate), target_rate) + waveform = resample_poly( + waveform, target_rate // divisor, int(sample_rate) // divisor + ).astype(np.float32) + wav_path = audio_root / f"{index:04d}.wav" + sf.write(wav_path, waveform, target_rate, subtype="PCM_16") + prepared.append(SpeechSample(sample_id, gold_text, waveform, target_rate, wav_path)) + return prepared + + +def write_result(output: Path, samples: Sequence[SpeechSample], texts: Sequence[str]) -> None: + if len(samples) != len(texts): + raise ValueError("speech reference returned a different sample count") + rows = [] + for sample, text in zip(samples, texts, strict=True): + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"speech reference returned an empty transcript for {sample.sample_id}") + rows.append( + { + "sample_id": sample.sample_id, + "text": text.strip(), + "audio_path": str(sample.wav_path), + } + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps({"samples": rows}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def required_string(value: Any, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + return value + + +def model_id(request: Mapping[str, Any]) -> tuple[str, str | None]: + model = required_string(request.get("model"), "model") + revision = request.get("revision") + if revision is not None and (not isinstance(revision, str) or not revision): + raise ValueError("revision must be a non-empty string when set") + return model, revision + + +def torch_dtype(torch_module: Any, request: Mapping[str, Any]) -> Any: + return { + "fp16": torch_module.float16, + "fp32": torch_module.float32, + "bf16": torch_module.bfloat16, + }[required_string(request.get("precision"), "precision")] diff --git a/qualification_tests/benchmark_qualification/references/timm_image_classification.py b/qualification_tests/benchmark_qualification/references/timm_image_classification.py new file mode 100644 index 0000000000..a917201a18 --- /dev/null +++ b/qualification_tests/benchmark_qualification/references/timm_image_classification.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Produce model-agnostic TIMM image-classification reference outputs.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Mapping, Sequence + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + value.add_argument("--request", required=True, type=Path) + value.add_argument("--output", required=True, type=Path) + return value + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = parser().parse_args(argv) + request = json.loads(arguments.request.read_text(encoding="utf-8")) + if not isinstance(request, Mapping): + raise ValueError("reference request must contain an object") + + import timm + import torch + from PIL import Image + from timm.data import create_transform, resolve_model_data_config + + model_id = _string(request.get("model"), "model") + revision = request.get("revision") + if revision is not None: + model_id += "@" + _string(revision, "revision") + dtype = { + "fp16": torch.float16, + "fp32": torch.float32, + "bf16": torch.bfloat16, + }[_string(request.get("precision"), "precision")] + batch_size = int(request.get("batch_size", 16)) + if batch_size < 1: + raise ValueError("batch_size must be positive") + samples = request.get("samples") + if not isinstance(samples, list) or not samples: + raise ValueError("samples must be a non-empty list") + + device = torch.device("cuda") + model = timm.create_model(f"hf-hub:{model_id}", pretrained=True) + model = model.eval().to(device=device, dtype=dtype) + transform = create_transform(**resolve_model_data_config(model), is_training=False) + outputs: list[dict[str, Any]] = [] + for start in range(0, len(samples), batch_size): + batch = samples[start : start + batch_size] + images = [] + for index, sample in enumerate(batch, start=start): + if not isinstance(sample, Mapping): + raise ValueError(f"sample {index} must be an object") + path = Path(_string(sample.get("image_path"), f"sample {index} image_path")) + with Image.open(path) as image: + images.append(transform(image.convert("RGB"))) + pixels = torch.stack(images).to(device=device, dtype=dtype) + with torch.inference_mode(): + classes = model(pixels).argmax(dim=-1).cpu().tolist() + outputs.extend( + { + "sample_id": _string(sample.get("sample_id"), "sample_id"), + "top_class": int(top_class), + } + for sample, top_class in zip(batch, classes, strict=True) + ) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps({"samples": outputs}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} must be a non-empty string") + return value + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_qualification/runtime.py b/qualification_tests/benchmark_qualification/runtime.py similarity index 67% rename from tools/benchmark_qualification/runtime.py rename to qualification_tests/benchmark_qualification/runtime.py index decc0537a7..5558643076 100644 --- a/tools/benchmark_qualification/runtime.py +++ b/qualification_tests/benchmark_qualification/runtime.py @@ -9,6 +9,7 @@ import html import json import os +import site import shutil import subprocess import sys @@ -21,6 +22,16 @@ REPOSITORY = Path(__file__).resolve().parents[2] +# Public benchmark requests use Task API field names. These legacy manifest +# tasks still describe their primary assets under ``inputs`` with shorter names. +_MANIFEST_INPUT_FIELDS = { + "robot_control": {"image_path": "image", "state_path": "state"}, + "stereo_disparity": { + "left_image_path": "left_image", + "right_image_path": "right_image", + }, +} + @dataclass(frozen=True) class RuntimeContext: @@ -92,23 +103,47 @@ def require_candidate(context: RuntimeContext) -> tuple[Path, Path]: def write_model_descriptor( - case: QualificationCase, output: Path, request: Mapping[str, Any] + case: QualificationCase, + output: Path, + request: Mapping[str, Any], + *, + context: RuntimeContext | None = None, ) -> Path: candidate = case.candidate task = str(candidate["task"]) testcase: dict[str, Any] = {"name": case.name} if task == "time_series_forecast": testcase["inputs"] = dict(request) + elif task in _MANIFEST_INPUT_FIELDS: + testcase.update(request) + fields = _MANIFEST_INPUT_FIELDS[task] + testcase["inputs"] = {target: testcase.pop(source) for source, target in fields.items()} else: testcase.update(request) + selected_task = candidate.get("selected_task") + if selected_task is not None: + testcase["selected_task"] = str(selected_task) + checkpoint = str(candidate["checkpoint"]) + configured_directory = candidate.get("model_directory") + if configured_directory is not None: + if context is None: + raise QualificationError("candidate.model_directory requires a runtime context") + environment = reference_python(case, context).parent.parent.resolve() + model_directory = (environment / str(configured_directory)).resolve() + if not model_directory.is_relative_to(environment) or not model_directory.is_dir(): + raise QualificationError( + f"prepared candidate model directory is unavailable: {model_directory}" + ) + checkpoint = str(model_directory) value = { "name": case.model, - "hf_id": str(candidate["checkpoint"]), + "hf_id": checkpoint, "hf_revision": str(candidate.get("revision", "")), "bundle": str(candidate.get("bundle", f"{case.model}.bundle")), "family": case.family, "task": task, "precision": str(candidate["precision"]), + "trust_remote_code": bool(candidate.get("trust_remote_code", False)), "testcases": [testcase], **dict(candidate.get("build", {})), } @@ -165,8 +200,12 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: if requirements is None: return Path(os.path.abspath(sys.executable)) digest = hashlib.sha256() + digest.update(b"qualification-family-environment-v1\0") digest.update(requirements.read_bytes()) digest.update(sys.version.encode()) + digest.update(f"build-isolation={case.reference_build_isolation}".encode()) + if case.environment_hook is not None: + digest.update(case.environment_hook.read_bytes()) environment = context.environment_root / f"{case.family}-{digest.hexdigest()[:12]}" python = environment / "bin/python" stamp = environment / ".qualification-requirements.sha256" @@ -176,6 +215,7 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: and stamp.is_file() and stamp.read_text(encoding="utf-8").strip() == expected ): + _inherit_parent_site_packages(environment) return python environment.parent.mkdir(parents=True, exist_ok=True) setup_root = context.artifacts / "environment-setup" / case.family @@ -189,27 +229,94 @@ def reference_python(case: QualificationCase, context: RuntimeContext) -> Path: ) if created.returncode != 0 or not python.is_file(): raise QualificationError(f"cannot create reference environment; see {setup_root}") + _inherit_parent_site_packages(environment) + install_command = [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + ] + if not case.reference_build_isolation: + install_command.append("--no-build-isolation") + install_command.extend(("-r", str(requirements))) + install_environment = dict(os.environ) + # Source distributions commonly delegate extension builds to Ninja. Keep one + # family environment from exhausting a shared validation host while allowing + # operators to select a different limit explicitly. + install_environment.setdefault("MAX_JOBS", "4") installed = run_command( - [ - str(python), - "-m", - "pip", - "install", - "--disable-pip-version-check", - "-r", - str(requirements), - ], + install_command, setup_root, "pip", - timeout=1800, + # Native reference dependencies such as FlashAttention can take more + # than 30 minutes to build on aarch64 even with bounded parallelism. + timeout=7200, verbose=context.verbose, + env=install_environment, ) if installed.returncode != 0: raise QualificationError(f"cannot install reference requirements; see {setup_root}") + if case.environment_hook is not None: + prepared = run_command( + [str(python), str(case.environment_hook)], + setup_root, + "prepare", + timeout=1800, + verbose=context.verbose, + ) + if prepared.returncode != 0: + raise QualificationError(f"cannot prepare family environment; see {setup_root}") stamp.write_text(expected + "\n", encoding="utf-8") return python +def reference_environment_paths(case: QualificationCase, context: RuntimeContext) -> dict[str, str]: + """Resolve family-declared reference inputs below its isolated environment.""" + if not case.reference_paths: + return {} + environment = reference_python(case, context).parent.parent.resolve() + result = {} + for name, relative in case.reference_paths.items(): + path = (environment / relative).resolve() + if not path.is_relative_to(environment) or not path.exists(): + raise QualificationError(f"prepared reference path {name!r} is unavailable: {path}") + result[name] = str(path) + return result + + +def reference_environment_options( + case: QualificationCase, + context: RuntimeContext, + configured: Mapping[str, Any], +) -> dict[str, Any]: + """Add family-declared reference paths without knowing their meaning.""" + result = dict(configured) + paths = reference_environment_paths(case, context) + if duplicates := sorted(result.keys() & paths.keys()): + raise QualificationError( + "reference options duplicate prepared environment paths: " + ", ".join(duplicates) + ) + result.update(paths) + return result + + +def _inherit_parent_site_packages(environment: Path) -> None: + child_packages = sorted(environment.glob("lib/python*/site-packages")) + if len(child_packages) != 1: + raise QualificationError( + f"reference environment has no unambiguous site-packages directory: {environment}" + ) + parent_packages = sorted( + {str(Path(value).resolve()) for value in site.getsitepackages() if Path(value).is_dir()} + ) + if not parent_packages or any("\n" in value for value in parent_packages): + raise QualificationError("cannot resolve parent Python site-packages") + (child_packages[0] / "trtmc-parent-environment.pth").write_text( + "\n".join(parent_packages) + "\n", encoding="utf-8" + ) + + def run_command( command: Sequence[str], output: Path, diff --git a/tools/benchmark_qualification/accuracy.py b/tools/benchmark_qualification/accuracy.py deleted file mode 100644 index 73a52799b7..0000000000 --- a/tools/benchmark_qualification/accuracy.py +++ /dev/null @@ -1,398 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Internal Accuracy qualification through the installed user benchmark.""" - -from __future__ import annotations - -import csv -import json -import math -import random -from pathlib import Path -from typing import Any, Mapping, Sequence - -from .catalog import QualificationCase, QualificationError, load_benchmark -from .datasets import Dataset, resolve_dataset -from .runtime import ( - RuntimeContext, - prepare_bundle, - reference_python, - require_candidate, - run_command, - write_model_descriptor, - write_result, -) - - -def run_accuracy(case: QualificationCase, context: RuntimeContext) -> dict[str, Any]: - output = context.case_artifacts(case) - output.mkdir(parents=True, exist_ok=True) - definition = load_benchmark(context.repository, case) - dataset = resolve_dataset(definition, context) - metric = definition.get("metric") - metric_name = metric.get("name") if isinstance(metric, Mapping) else None - if metric_name == "exact_token_ids": - result = _mmlu(case, context, dataset, output) - elif metric_name == "forecast_tensor_parity": - result = _etth1(case, context, definition, dataset, output) - else: - raise QualificationError(f"unsupported Accuracy metric {metric_name!r}") - write_result(output, result) - return result - - -def _mmlu( - case: QualificationCase, - context: RuntimeContext, - dataset: Dataset, - output: Path, -) -> dict[str, Any]: - configured = case.values - sample_limit = _positive_int(configured.get("samples"), "accuracy.samples") - payload = json.loads(dataset.path.read_text(encoding="utf-8")) - requests = payload.get("requests") if isinstance(payload, Mapping) else None - if not isinstance(requests, list) or not requests: - raise QualificationError("MMLU dataset must contain a non-empty requests list") - selected = [] - for index, request in enumerate(requests[:sample_limit]): - if not isinstance(request, Mapping): - raise QualificationError(f"MMLU request {index} must be an object") - selected.append( - { - "sample_id": str(request.get("id") or request.get("sample_id") or f"mmlu-{index}"), - "prompt": _prompt(request), - } - ) - reference = configured.get("reference", {}) - candidate_request = configured.get("request", {}) - if not isinstance(reference, Mapping) or not isinstance(candidate_request, Mapping): - raise QualificationError("MMLU reference and request must be objects") - reference_request = { - "model": str(case.candidate["checkpoint"]), - "revision": case.candidate.get("revision"), - "precision": str(reference.get("precision", "fp32")), - "prompt_token_limit": int(configured.get("prompt_token_limit", 192)), - "truncation_side": str(configured.get("truncation_side", "left")), - "generation": dict(candidate_request), - "samples": selected, - } - request_path = output / "reference-request.json" - reference_path = output / "reference.json" - _json(request_path, reference_request) - runner = context.repository / "tools/benchmark_qualification/references/hf_text_generation.py" - completed = run_command( - [ - str(reference_python(case, context)), - str(runner), - "--request", - str(request_path), - "--output", - str(reference_path), - ], - output, - "reference", - timeout=3600, - verbose=context.verbose, - ) - if completed.returncode != 0: - raise QualificationError(f"HF Accuracy reference failed; see {output}") - reference_samples = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") - if not isinstance(reference_samples, list) or len(reference_samples) != len(selected): - raise QualificationError("HF Accuracy reference returned an invalid sample set") - - candidate_requests = [ - { - "sample_id": sample["sample_id"], - "request": {**dict(candidate_request), "prompt": sample["prompt"]}, - } - for sample in reference_samples - ] - candidate, bundle = _candidate_outputs(case, context, output, "generate", candidate_requests) - rows = [] - for actual, expected in zip(candidate, reference_samples, strict=True): - actual_ids = actual.get("token_ids") - expected_ids = expected.get("token_ids") - matched = actual_ids == expected_ids and isinstance(actual_ids, list) and bool(actual_ids) - rows.append( - { - "sample_id": expected["sample_id"], - "passed": matched, - "candidate_token_ids": actual_ids, - "reference_token_ids": expected_ids, - } - ) - gates = configured.get("gate", {}) - if not isinstance(gates, Mapping): - raise QualificationError("MMLU gate must be an object") - minimum_rate = float(gates.get("min_pass_rate", 1.0)) - allowed_failures = int(gates.get("allowed_failures", 0)) - passed_count = sum(bool(row["passed"]) for row in rows) - failed_count = len(rows) - passed_count - pass_rate = passed_count / len(rows) - passed = pass_rate >= minimum_rate and failed_count <= allowed_failures - return { - "schema_version": "trtmc.qualification-result/v1", - "case": case.id, - "kind": "accuracy", - "status": "passed" if passed else "failed", - "model": case.model, - "benchmark": case.benchmark, - "bundle": str(bundle), - "dataset": dataset.receipt(), - "metrics": { - "samples": len(rows), - "passed_samples": passed_count, - "failed_samples": failed_count, - "continuation_token_pass_rate": pass_rate, - }, - "gate": {"min_pass_rate": minimum_rate, "allowed_failures": allowed_failures}, - "samples": rows, - } - - -def _etth1( - case: QualificationCase, - context: RuntimeContext, - definition: Mapping[str, Any], - dataset: Dataset, - output: Path, -) -> dict[str, Any]: - configured = case.values - window = configured.get("window") - reference = configured.get("reference") - if not isinstance(window, Mapping) or not isinstance(reference, Mapping): - raise QualificationError("ETTh1 window and reference must be objects") - samples = _etth1_windows(dataset.path, definition, window, int(configured.get("samples", 10))) - runner_value = reference.get("command") - if not isinstance(runner_value, str) or not runner_value: - raise QualificationError("ETTh1 reference.command must be set") - runner = (case.source.parent / runner_value).resolve() - if not runner.is_file(): - raise QualificationError(f"ETTh1 reference runner does not exist: {runner}") - request = { - "model": str(case.candidate["checkpoint"]), - "revision": case.candidate.get("revision"), - "precision": str(reference.get("precision", "fp32")), - "samples": samples, - } - request_path = output / "reference-request.json" - reference_path = output / "reference.json" - _json(request_path, request) - completed = run_command( - [ - str(reference_python(case, context)), - str(runner), - "--request", - str(request_path), - "--output", - str(reference_path), - ], - output, - "reference", - timeout=3600, - verbose=context.verbose, - ) - if completed.returncode != 0: - raise QualificationError(f"ETTh1 Accuracy reference failed; see {output}") - expected = json.loads(reference_path.read_text(encoding="utf-8")).get("samples") - if not isinstance(expected, list) or len(expected) != len(samples): - raise QualificationError("ETTh1 reference returned an invalid sample set") - candidate_requests = [ - { - "sample_id": sample["sample_id"], - "request": { - "past_values": [float(value) for value in sample["past_values"]], - "observed_mask": [1.0] * len(sample["past_values"]), - "frequency": int(sample.get("frequency", 0)), - }, - } - for sample in samples - ] - actual, bundle = _candidate_outputs(case, context, output, "solve", candidate_requests) - gate = configured.get("gate", {}) - if not isinstance(gate, Mapping): - raise QualificationError("ETTh1 gate must be an object") - relative_limit = float(gate["max_relative_l2"]) - absolute_limit = float(gate["max_absolute_error"]) - rows = [] - for left, right in zip(actual, expected, strict=True): - left_values = [float(value) for value in left["values"]] - right_values = [float(value) for value in right["values"]] - shape_match = left.get("shape") == right.get("shape") and len(left_values) == len( - right_values - ) - if shape_match: - difference = math.sqrt( - sum((a - b) ** 2 for a, b in zip(left_values, right_values, strict=True)) - ) - norm = math.sqrt(sum(value**2 for value in right_values)) - relative_l2 = difference / max(norm, 1.0e-12) - maximum = max(abs(a - b) for a, b in zip(left_values, right_values, strict=True)) - else: - relative_l2 = math.inf - maximum = math.inf - rows.append( - { - "sample_id": right["sample_id"], - "passed": shape_match - and relative_l2 <= relative_limit - and maximum <= absolute_limit, - "shape_match": shape_match, - "relative_l2": relative_l2, - "max_absolute_error": maximum, - } - ) - passed = all(row["passed"] for row in rows) - return { - "schema_version": "trtmc.qualification-result/v1", - "case": case.id, - "kind": "accuracy", - "status": "passed" if passed else "failed", - "model": case.model, - "benchmark": case.benchmark, - "bundle": str(bundle), - "dataset": dataset.receipt(), - "metrics": { - "samples": len(rows), - "failed_samples": sum(not row["passed"] for row in rows), - "max_relative_l2": max(row["relative_l2"] for row in rows), - "max_absolute_error": max(row["max_absolute_error"] for row in rows), - }, - "gate": { - "max_relative_l2": relative_limit, - "max_absolute_error": absolute_limit, - }, - "samples": rows, - } - - -def _candidate_outputs( - case: QualificationCase, - context: RuntimeContext, - output: Path, - operation: str, - requests: Sequence[Mapping[str, Any]], -) -> tuple[list[dict[str, Any]], Path]: - if not requests: - raise QualificationError("Accuracy candidate has no requests") - worker, runtime_root = require_candidate(context) - first_request = requests[0].get("request") - if not isinstance(first_request, Mapping): - raise QualificationError("Accuracy candidate request must be an object") - descriptor = write_model_descriptor(case, output, first_request) - bundle = prepare_bundle(case, context, output, descriptor) - data_path = output / "candidate-inputs.json" - _json(data_path, list(requests)) - candidate_output = output / "candidate-run" - command = [ - str(context.trtmc_bench), - "run", - "--model", - str(descriptor), - "--case", - case.name, - "--operation", - operation, - "--bundle", - str(bundle), - "--data", - str(data_path), - "--runtime-root", - str(runtime_root), - "--worker", - str(worker), - "--warmup", - "0", - "--iterations", - "1", - "--telemetry", - "off", - "--no-build", - "--output", - str(candidate_output), - ] - completed = run_command( - command, - output, - "candidate", - timeout=7200, - verbose=context.verbose, - ) - if completed.returncode != 0: - raise QualificationError(f"TRTMC Accuracy candidate failed; see {output}") - result = json.loads((candidate_output / "result.json").read_text(encoding="utf-8")) - cells = result.get("cells") - if not isinstance(cells, list) or len(cells) != len(requests): - raise QualificationError("trtmc-bench returned an invalid Accuracy result set") - outputs = [] - for cell in cells: - if not isinstance(cell, Mapping) or cell.get("status") != "completed": - raise QualificationError("trtmc-bench reported a failed Accuracy request") - summary = cell.get("output_summary") - if not isinstance(summary, Mapping): - raise QualificationError("trtmc-bench omitted an Accuracy output") - outputs.append(dict(summary)) - return outputs, bundle - - -def _etth1_windows( - path: Path, - definition: Mapping[str, Any], - configured: Mapping[str, Any], - count: int, -) -> list[dict[str, Any]]: - with path.open(newline="", encoding="utf-8") as stream: - rows = list(csv.DictReader(stream)) - columns = configured.get("columns", ["OT"]) - if not isinstance(columns, list) or not columns: - raise QualificationError("ETTh1 window.columns must be a non-empty list") - context = int(configured.get("context_length", 512)) - prediction = int(configured.get("prediction_length", 64)) - start = int(configured.get("test_target_start", 11520)) - end = int(configured.get("test_end", 14400)) - stride = int(configured.get("stride", 24)) - starts = list(range(start - context, end - context - prediction + 1, stride)) - seed = int(definition.get("selection", {}).get("seed", 20260715)) - random.Random(seed).shuffle(starts) - if count < 1 or len(starts) < count or len(rows) < end: - raise QualificationError("ETTh1 dataset cannot satisfy the configured windows") - return [ - { - "sample_id": f"etth1-{index:04d}", - "past_values": [ - float(row[column]) - for row in rows[window_start : window_start + context] - for column in columns - ], - "frequency": int(configured.get("frequency", 0)), - } - for index, window_start in enumerate(starts[:count]) - ] - - -def _prompt(request: Mapping[str, Any]) -> str: - messages = request.get("messages") - if isinstance(messages, list): - for message in reversed(messages): - if ( - isinstance(message, Mapping) - and message.get("role") == "user" - and isinstance(message.get("content"), str) - and message["content"].strip() - ): - return str(message["content"]) - prompt = request.get("prompt") - if isinstance(prompt, str) and prompt.strip(): - return prompt - raise QualificationError("MMLU request has neither a prompt nor a user message") - - -def _positive_int(value: Any, name: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 1: - raise QualificationError(f"{name} must be a positive integer") - return value - - -def _json(path: Path, value: Any) -> None: - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/tools/benchmark_qualification/performance.py b/tools/benchmark_qualification/performance.py deleted file mode 100644 index a88f7ce732..0000000000 --- a/tools/benchmark_qualification/performance.py +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Internal Performance qualification through installed trtmc-bench.""" - -from __future__ import annotations - -import json -import os -import sys -from typing import Any, Mapping - -import yaml - -from .catalog import QualificationCase, QualificationError, load_benchmark -from .runtime import ( - RuntimeContext, - reference_python, - require_candidate, - run_command, - write_model_descriptor, - write_result, -) - - -def run_performance(case: QualificationCase, context: RuntimeContext) -> dict[str, Any]: - output = context.case_artifacts(case) - output.mkdir(parents=True, exist_ok=True) - definition = load_benchmark(context.repository, case) - worker, runtime_root = require_candidate(context) - configured = case.values - request = configured.get("request") - baseline = configured.get("reference") - measurement = configured.get("measurement") - reference_timing = definition.get("reference_timing") - if not all( - isinstance(value, Mapping) - for value in (request, baseline, measurement, reference_timing) - ): - raise QualificationError( - "Performance request, reference, measurement, and reference timing must be objects" - ) - assert isinstance(request, Mapping) - assert isinstance(baseline, Mapping) - assert isinstance(measurement, Mapping) - assert isinstance(reference_timing, Mapping) - descriptor = write_model_descriptor(case, output, request) - entry_id = f"qualification.{case.family}.{case.name}" - suite = { - "schema_version": "trtmc.perf-suite/v2", - "name": case.id, - "entries": [ - { - "id": entry_id, - "family": case.family, - "operation": str(configured["operation"]), - "model": case.model, - "manifest": str(descriptor), - "workload": { - "testcase": case.name, - "request": dict(request), - }, - "measurement": { - "warmup": int(measurement.get("warmup", 5)), - "iterations": int(measurement.get("iterations", 10)), - }, - "baseline": {**dict(baseline), **dict(reference_timing)}, - "equivalence_margin_percent": float( - configured.get("equivalence_margin_percent", 5.0) - ), - } - ], - } - if definition.get("stability") != { - "samples": 10, - "max_half_median_change_percent": 5.0, - "median_band_percent": 5.0, - "minimum_samples_within_band": 8, - "retries": 1, - }: - raise QualificationError("unsupported Performance stability definition") - suite_path = output / "resolved-suite.yaml" - suite_path.write_text(yaml.safe_dump(suite, sort_keys=False), encoding="utf-8") - results_root = output / "matrix" - references = { - "elf_repo": "", - "lance_repo": "", - "lerobot_repo": "", - "sana_repo": "", - "sana_model": "", - "personaplex_repo": "", - "fast_foundation_stereo_model": "", - } - environment = { - "schema_version": "trtmc.perf-environment/v2", - "name": "qualification", - "tools": { - "trtmc_bench": str(context.trtmc_bench), - "trtmc_worker": str(worker), - "hf_transformers_runner": str( - context.repository / "apps/benchmark/performance/baselines/hf_transformers.py" - ), - "task_reference_runner": str( - context.repository / "apps/benchmark/performance/baselines/task_reference.py" - ), - "reference_python": str(reference_python(case, context)), - }, - "references": references, - "storage": { - "results_root": str(results_root), - "scratch_root": str(output / "scratch"), - "bundle_cache": str(context.bundle_cache), - "bundle_roots": [str(path) for path in context.bundle_roots], - "runtime_root": str(runtime_root), - "bundle_retention": "retain", - }, - "execution": { - "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1", - "hf_cache_mode": "shared", - "hf_cache_retention": "retain", - "timeout_seconds": int(configured.get("timeout_seconds", 7200)), - }, - } - environment_path = output / "resolved-environment.yaml" - environment_path.write_text(yaml.safe_dump(environment, sort_keys=False), encoding="utf-8") - command = [ - sys.executable, - str(context.repository / "tools/perf_matrix.py"), - "run", - str(suite_path), - "--environment", - str(environment_path), - "--entry", - entry_id, - "--allow-partial", - ] - if context.no_build: - command.append("--no-build") - if context.verbose: - command.append("--verbose") - completed = run_command( - command, - output, - "performance", - timeout=int(configured.get("timeout_seconds", 7200)) * 2, - verbose=context.verbose, - ) - run_directories = sorted(path.parent for path in results_root.glob("*/results.json")) - if not run_directories: - raise QualificationError(f"Performance produced no result; see {output}") - run_directory = run_directories[-1] - matrix = json.loads((run_directory / "results.json").read_text(encoding="utf-8")) - rows = matrix.get("rows") - row = rows[0] if isinstance(rows, list) and len(rows) == 1 else {} - passed = completed.returncode == 0 and matrix.get("status") == "completed" - comparison = row.get("comparison", {}) if isinstance(row, Mapping) else {} - result = { - "schema_version": "trtmc.qualification-result/v1", - "case": case.id, - "kind": "performance", - "status": "passed" if passed else "failed", - "model": case.model, - "benchmark": case.benchmark, - "matrix_run": str(run_directory), - "comparison_status": row.get("status") if isinstance(row, Mapping) else None, - "metrics": { - "candidate_p50_ms": comparison.get("candidate_p50_ms"), - "reference_p50_ms": comparison.get("reference_p50_ms"), - "reference_over_candidate_p50": comparison.get("reference_over_candidate_p50"), - }, - "reference_attempts": row.get("reference_attempts", []) if isinstance(row, Mapping) else [], - } - write_result(output, result) - return result diff --git a/tools/benchmark_qualification/references/hf_text_generation.py b/tools/benchmark_qualification/references/hf_text_generation.py deleted file mode 100644 index 5f6d3af657..0000000000 --- a/tools/benchmark_qualification/references/hf_text_generation.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Produce deterministic Hugging Face text-generation qualification references.""" - -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path -from typing import Any, Mapping - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--request", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - arguments = parser.parse_args() - request = json.loads(arguments.request.read_text(encoding="utf-8")) - - import torch - from transformers import AutoModelForCausalLM, AutoTokenizer - - if not torch.cuda.is_available(): - raise RuntimeError("the HF Accuracy reference requires CUDA") - precision = str(request.get("precision", "fp32")) - dtypes = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16} - if precision not in dtypes: - raise ValueError(f"unsupported reference precision {precision!r}") - common: dict[str, Any] = { - "local_files_only": os.environ.get("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY") == "1" - } - if request.get("revision"): - common["revision"] = request["revision"] - tokenizer = AutoTokenizer.from_pretrained(str(request["model"]), **common) - if tokenizer.pad_token_id is None: - tokenizer.pad_token = tokenizer.eos_token - model = AutoModelForCausalLM.from_pretrained( - str(request["model"]), dtype=dtypes[precision], **common - ).eval() - model.to("cuda") - generation = request.get("generation", {}) - if not isinstance(generation, Mapping): - raise ValueError("generation must be an object") - results = [] - with torch.inference_mode(): - for sample in request["samples"]: - prompt = _truncate( - tokenizer, - str(sample["prompt"]), - int(request["prompt_token_limit"]), - str(request.get("truncation_side", "left")), - ) - encoded = tokenizer(prompt, return_tensors="pt") - encoded = {name: value.to("cuda") for name, value in encoded.items()} - options: dict[str, Any] = { - "max_new_tokens": int(generation.get("max_new_tokens", 64)), - "do_sample": bool(generation.get("do_sample", False)), - "num_beams": 1, - "pad_token_id": tokenizer.eos_token_id, - "eos_token_id": tokenizer.eos_token_id, - "repetition_penalty": float(generation.get("repetition_penalty", 1.0)), - } - if options["do_sample"]: - options.update( - temperature=float(generation.get("temperature", 1.0)), - top_k=int(generation.get("top_k", 0)), - top_p=float(generation.get("top_p", 1.0)), - ) - seed = int(generation.get("seed", -1)) - if seed >= 0: - torch.manual_seed(seed) - generated = model.generate(**encoded, **options) - prompt_length = int(encoded["input_ids"].shape[-1]) - token_ids = generated[0, prompt_length:].detach().cpu().tolist() - results.append( - { - "sample_id": str(sample["sample_id"]), - "prompt": prompt, - "token_ids": [int(value) for value in token_ids], - "text": tokenizer.decode(token_ids, skip_special_tokens=True), - } - ) - output = { - "schema_version": "trtmc.accuracy-reference/v1", - "model": request["model"], - "revision": getattr(model.config, "_commit_hash", None) or request.get("revision"), - "precision": precision, - "samples": results, - } - arguments.output.write_text( - json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - return 0 - - -def _truncate(tokenizer: Any, prompt: str, limit: int, side: str) -> str: - if limit < 1 or side not in {"left", "right"}: - raise ValueError("prompt token limit and truncation side are invalid") - token_ids = [int(value) for value in tokenizer.encode(prompt, add_special_tokens=False)] - if len(token_ids) <= limit: - return prompt - selected = token_ids[-limit:] if side == "left" else token_ids[:limit] - return str( - tokenizer.decode( - selected, - skip_special_tokens=False, - clean_up_tokenization_spaces=False, - ) - ) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/model_benchmark.py b/tools/model_benchmark.py index d2e8708bb0..80abf73b18 100644 --- a/tools/model_benchmark.py +++ b/tools/model_benchmark.py @@ -21,15 +21,15 @@ if str(REPOSITORY) not in sys.path: sys.path.insert(0, str(REPOSITORY)) -from tools.benchmark_qualification.accuracy import run_accuracy # noqa: E402 -from tools.benchmark_qualification.catalog import ( # noqa: E402 +from qualification_tests.benchmark_qualification.accuracy import run_accuracy # noqa: E402 +from qualification_tests.benchmark_qualification.catalog import ( # noqa: E402 QualificationCase, QualificationError, discover, select, ) -from tools.benchmark_qualification.performance import run_performance # noqa: E402 -from tools.benchmark_qualification.runtime import context_from_args, write_result # noqa: E402 +from qualification_tests.benchmark_qualification.performance.qualification import run_performance # noqa: E402 +from qualification_tests.benchmark_qualification.runtime import context_from_args, write_result # noqa: E402 def parser() -> argparse.ArgumentParser: @@ -126,13 +126,13 @@ def _run(cases: Sequence[QualificationCase], arguments: argparse.Namespace) -> i "cases": results, } _write_summary(context.artifacts, summary) - print(f"JSON: {context.artifacts / 'summary.json'}") + print(f"JSON: {context.artifacts / 'report.json'}") print(f"HTML: {context.artifacts / 'report.html'}") return 0 if summary["status"] == "passed" else 1 def _write_summary(output: Path, summary: Mapping[str, Any]) -> None: - (output / "summary.json").write_text( + (output / "report.json").write_text( json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) rows = [] @@ -148,7 +148,7 @@ def _write_summary(output: Path, summary: Mapping[str, Any]) -> None: document = """TRTMC model benchmark

Internal model benchmark

Status: {status}

-{rows}
ModelKindCaseStatus

summary.json

+{rows}

report.json

""".format(status=html.escape(str(summary["status"])), rows="".join(rows)) (output / "report.html").write_text(document, encoding="utf-8") diff --git a/tools/prepare_coco_detection_dataset.py b/tools/prepare_coco_detection_dataset.py new file mode 100644 index 0000000000..b976d9cc41 --- /dev/null +++ b/tools/prepare_coco_detection_dataset.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare a deterministic COCO val2017 object-detection subset.""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +import json +import shutil +from pathlib import Path +from typing import Any, Mapping, Sequence + + +def _positive_int(raw: str) -> int: + value = int(raw) + if value < 1: + raise argparse.ArgumentTypeError("value must be positive") + return value + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--coco-root", + required=True, + type=Path, + help="Directory containing annotations/instances_val2017.json and val2017/.", + ) + parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument("--limit", default=100, type=_positive_int) + return parser + + +def _objects(raw: Any, label: str) -> list[dict[str, Any]]: + if not isinstance(raw, list) or not all(isinstance(value, Mapping) for value in raw): + raise ValueError(f"COCO {label} must be a list of objects") + return [dict(value) for value in raw] + + +def _valid_annotation(annotation: Mapping[str, Any]) -> bool: + bbox = annotation.get("bbox") + return ( + not bool(annotation.get("iscrowd", 0)) + and isinstance(bbox, list) + and len(bbox) == 4 + and float(bbox[2]) > 0.0 + and float(bbox[3]) > 0.0 + and float(annotation.get("area", float(bbox[2]) * float(bbox[3]))) > 0.0 + ) + + +def _select_images( + images: Sequence[Mapping[str, Any]], + annotations: Sequence[Mapping[str, Any]], + category_ids: Sequence[int], + limit: int, +) -> list[int]: + image_by_id = {int(image["id"]): image for image in images} + by_category: dict[int, list[int]] = defaultdict(list) + seen_pairs: set[tuple[int, int]] = set() + for annotation in sorted(annotations, key=lambda value: int(value["id"])): + if not _valid_annotation(annotation): + continue + image_id = int(annotation["image_id"]) + category_id = int(annotation["category_id"]) + image = image_by_id.get(image_id) + if image is None or category_id not in category_ids: + continue + bbox = annotation["bbox"] + image_area = float(image["width"]) * float(image["height"]) + box_area = float(annotation.get("area", float(bbox[2]) * float(bbox[3]))) + pair = (category_id, image_id) + if box_area >= image_area * 0.01 and pair not in seen_pairs: + by_category[category_id].append(image_id) + seen_pairs.add(pair) + + selected: list[int] = [] + selected_set: set[int] = set() + offset = 0 + while len(selected) < limit: + made_progress = False + for category_id in category_ids: + rows = by_category.get(category_id, []) + if offset >= len(rows): + continue + made_progress = True + image_id = rows[offset] + if image_id in selected_set: + continue + selected.append(image_id) + selected_set.add(image_id) + if len(selected) == limit: + return selected + if not made_progress: + break + offset += 1 + raise ValueError(f"COCO annotations provide only {len(selected)} eligible unique images") + + +def prepare(coco_root: Path, output_root: Path, limit: int) -> Path: + annotation_path = coco_root / "annotations" / "instances_val2017.json" + image_root = coco_root / "val2017" + payload = json.loads(annotation_path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise ValueError("COCO annotation file must contain an object") + images = _objects(payload.get("images"), "images") + annotations = _objects(payload.get("annotations"), "annotations") + categories = sorted( + _objects(payload.get("categories"), "categories"), key=lambda value: int(value["id"]) + ) + category_ids = [int(category["id"]) for category in categories] + category_indices = {category_id: index for index, category_id in enumerate(category_ids)} + selected_ids = _select_images(images, annotations, category_ids, limit) + selected_set = set(selected_ids) + image_by_id = {int(image["id"]): image for image in images} + annotations_by_image: dict[int, list[dict[str, Any]]] = defaultdict(list) + for annotation in annotations: + image_id = int(annotation["image_id"]) + if image_id not in selected_set or not _valid_annotation(annotation): + continue + x, y, width, height = (float(value) for value in annotation["bbox"]) + category_id = int(annotation["category_id"]) + annotations_by_image[image_id].append( + { + "id": int(annotation["id"]), + "category_id": category_id, + "category_index": category_indices[category_id], + "bbox_xyxy": [x, y, x + width, y + height], + "area": float(annotation.get("area", width * height)), + } + ) + + output_dir = output_root / "COCO2017_object_detection" + requests = [] + for image_id in selected_ids: + image = image_by_id[image_id] + filename = str(image["file_name"]) + relative = Path("images") / filename + source = image_root / filename + if not source.is_file(): + raise FileNotFoundError(f"COCO image is unavailable: {source}") + destination = output_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + requests.append( + { + "id": f"coco2017_{image_id:012d}", + "image": relative.as_posix(), + "image_id": image_id, + "width": int(image["width"]), + "height": int(image["height"]), + "annotations": sorted( + annotations_by_image[image_id], key=lambda value: int(value["id"]) + ), + } + ) + + manifest = { + "dataset": "COCO 2017 validation object detection", + "sources": [ + "http://images.cocodataset.org/zips/val2017.zip", + "http://images.cocodataset.org/annotations/annotations_trainval2017.zip", + ], + "sampling": ( + "deterministic category-balanced round-robin over unique images; " + "each selected category has a non-crowd instance covering at least 1% of the image" + ), + "label_spaces": { + "coco-category-id": "category_id", + "coco-contiguous-80": "category_index", + }, + "categories": [ + { + "id": int(category["id"]), + "index": category_indices[int(category["id"])], + "name": str(category["name"]), + } + for category in categories + ], + "requests": requests, + } + output = output_dir / "coco2017_object_detection.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return output + + +def main() -> int: + arguments = _parser().parse_args() + output = prepare( + arguments.coco_root.resolve(), arguments.output_root.resolve(), arguments.limit + ) + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/test_impact.py b/tools/test_impact.py index 254a628dad..090794ea1f 100644 --- a/tools/test_impact.py +++ b/tools/test_impact.py @@ -24,7 +24,7 @@ "README.md", } MODEL_PROOF_NEUTRAL_FILES = { - "apps/benchmark/performance/release.yaml", + "qualification_tests/benchmark_qualification/performance/config/release.yaml", } SHARED_PREFIXES = ( ".github/", @@ -33,6 +33,7 @@ "core/", "examples/", "plugins/", + "qualification_tests/", "requirements/", "third_party/", "tools/", diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 0c889fd104..e807bc9fe4 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -508,20 +508,63 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/legal_headers.py", "tools/model_ci.py", "tools/model_benchmark.py", - "tools/perf_matrix.py", "tools/pr_metadata.py", "tools/test_impact.py", - "tools/benchmark_qualification/__init__.py", - "tools/benchmark_qualification/accuracy.py", - "tools/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", - "tools/benchmark_qualification/benchmarks/mmlu_continuation.yaml", - "tools/benchmark_qualification/benchmarks/text_generation_performance.yaml", - "tools/benchmark_qualification/benchmarks/time_series_performance.yaml", - "tools/benchmark_qualification/catalog.py", - "tools/benchmark_qualification/datasets.py", - "tools/benchmark_qualification/performance.py", - "tools/benchmark_qualification/references/hf_text_generation.py", - "tools/benchmark_qualification/runtime.py", + "qualification_tests/benchmark_qualification/__init__.py", + "qualification_tests/benchmark_qualification/accuracy.py", + "qualification_tests/benchmark_qualification/benchmarks/audio_generation_model_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/audio_generation_pipeline_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/beans_image_feature_knn.yaml", + "qualification_tests/benchmark_qualification/benchmarks/embedding_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/encoder_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/etth1_forecast_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_seq2seq.yaml", + "qualification_tests/benchmark_qualification/benchmarks/flores200_en_fr_translation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/humaneval_continuation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/image_classification_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/image_features_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_classification.yaml", + "qualification_tests/benchmark_qualification/benchmarks/coco2017_object_detection.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_localization_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_prompted_segmentation_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_segmentation_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_text_prompted_instance_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/imagenette_vision_language_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/librispeech_transcription.yaml", + "qualification_tests/benchmark_qualification/benchmarks/media_generation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/metric_geometry_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/metric_geometry_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/mmlu_continuation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/object_detection_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/ocrbench_v2_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/reranking_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/robot_action_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/robot_action_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/newstest2019_en_ru_translation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_embedding_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stsbenchmark_reranking_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/task_output_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/speech_nemo_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/speech_transformer_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_parity.yaml", + "qualification_tests/benchmark_qualification/benchmarks/stereo_disparity_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/text_generation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/text_prompted_segmentation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/vision_language_pipeline_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/vision_language_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/semantic_segmentation_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/time_series_performance.yaml", + "qualification_tests/benchmark_qualification/benchmarks/wikitext103_continuation.yaml", + "qualification_tests/benchmark_qualification/benchmarks/wmt14_en_de_translation.yaml", + "qualification_tests/benchmark_qualification/catalog.py", + "qualification_tests/benchmark_qualification/datasets.py", + "qualification_tests/benchmark_qualification/performance/qualification.py", + "qualification_tests/benchmark_qualification/references/hf_encoder.py", + "qualification_tests/benchmark_qualification/references/hf_text_generation.py", + "qualification_tests/benchmark_qualification/references/speech_io.py", + "qualification_tests/benchmark_qualification/references/timm_image_classification.py", + "qualification_tests/benchmark_qualification/runtime.py", + "tools/prepare_coco_detection_dataset.py", "tools/ci/__init__.py", "tools/ci/__main__.py", "tools/ci/container.py", @@ -535,6 +578,10 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/ci/quality.py", "tools/ci/stage.py", } + migrated_qualification = { + path for path in expected_tools if path.startswith("qualification_tests/") + } + expected_tools -= migrated_qualification expected_tool_tests = { "tools/tests/__init__.py", "tools/tests/test_architecture.py", @@ -551,6 +598,39 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/tests/test_pr_metadata.py", "tools/tests/test_public_source_hygiene.py", } + expected_qualification = migrated_qualification | { + "qualification_tests/__init__.py", + "qualification_tests/benchmark_qualification/__init__.py", + "qualification_tests/benchmark_qualification/performance/__init__.py", + "qualification_tests/benchmark_qualification/performance/__main__.py", + "qualification_tests/benchmark_qualification/performance/README.md", + "qualification_tests/benchmark_qualification/performance/config/environments/auto-thor.yaml", + "qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml", + "qualification_tests/benchmark_qualification/performance/config/environments/l4t-thor.yaml", + "qualification_tests/benchmark_qualification/performance/config/release.yaml", + "qualification_tests/benchmark_qualification/performance/config/requirements.txt", + "qualification_tests/benchmark_qualification/performance/matrix.py", + "qualification_tests/benchmark_qualification/performance/reference_harness.py", + "qualification_tests/benchmark_qualification/performance/reference_protocol.py", + "qualification_tests/benchmark_qualification/performance/reporting.py", + "qualification_tests/benchmark_qualification/performance/references/__init__.py", + "qualification_tests/benchmark_qualification/performance/references/audio_reference.py", + "qualification_tests/benchmark_qualification/performance/references/hf_transformers.py", + "qualification_tests/benchmark_qualification/performance/references/generic_reference.py", + "qualification_tests/benchmark_qualification/performance/references/timing_contracts.py", + "qualification_tests/benchmark_qualification/performance/runner.py", + "qualification_tests/benchmark_qualification/performance/stability.py", + "qualification_tests/benchmark_qualification/performance/types.py", + "qualification_tests/benchmark_qualification/performance/tests/__init__.py", + "qualification_tests/benchmark_qualification/performance/tests/test_diffusers_semantic_reference.py", + "qualification_tests/benchmark_qualification/performance/tests/test_family_performance.py", + "qualification_tests/benchmark_qualification/performance/tests/test_offline_speech_contract.py", + "qualification_tests/benchmark_qualification/performance/tests/test_perf_matrix.py", + "qualification_tests/benchmark_qualification/performance/tests/test_reference_protocol.py", + "qualification_tests/benchmark_qualification/performance/tests/test_reporting.py", + "qualification_tests/benchmark_qualification/performance/tests/test_structured_output_contracts.py", + "qualification_tests/benchmark_qualification/performance/tests/test_timing_contracts.py", + } expected_cmake = {"cmake/trtmcConfig.cmake.in"} expected_third_party = { "third_party/stb/stb_image.h", @@ -572,6 +652,7 @@ def files(root: str) -> set[str]: tool_tests = {path for path in tool_files if path.startswith("tools/tests/")} assert tool_files - tool_tests == expected_tools assert tool_tests == expected_tool_tests + assert files("qualification_tests") == expected_qualification assert files("cmake") == expected_cmake assert files("third_party") == expected_third_party @@ -623,7 +704,7 @@ def test_applications_depend_only_on_public_model_connect_surfaces() -> None: REPO / "examples", ) application_files = [path for root in application_roots for path in root.rglob("*")] + [ - REPO / "tools/perf_matrix.py" + REPO / "qualification_tests/benchmark_qualification/performance/matrix.py" ] for path in application_files: if not path.is_file() or "__pycache__" in path.parts: diff --git a/tools/tests/test_e2e_evidence.py b/tools/tests/test_e2e_evidence.py index 2cc20a575d..917af49ead 100644 --- a/tools/tests/test_e2e_evidence.py +++ b/tools/tests/test_e2e_evidence.py @@ -1554,7 +1554,7 @@ def test_demo_shared_chart_keeps_scope_and_rejects_invalid_values() -> None: "ocr", ], ) -def test_text_task_references_are_visible_without_opening_details( +def test_text_generic_references_are_visible_without_opening_details( tmp_path: Path, task: str ) -> None: data = _case() diff --git a/tools/tests/test_family_impact.py b/tools/tests/test_family_impact.py index 2f6b395527..5cb0ac3279 100644 --- a/tools/tests/test_family_impact.py +++ b/tools/tests/test_family_impact.py @@ -49,7 +49,7 @@ def test_release_performance_policy_does_not_expand_family_scope(tmp_path: Path) impact = test_impact.classify( repo, [ - "apps/benchmark/performance/release.yaml", + "qualification_tests/benchmark_qualification/performance/config/release.yaml", "families/alpha/model.py", ], ) diff --git a/tools/tests/test_model_benchmark.py b/tools/tests/test_model_benchmark.py index 740eb50b9e..6da332fe2d 100644 --- a/tools/tests/test_model_benchmark.py +++ b/tools/tests/test_model_benchmark.py @@ -3,50 +3,2029 @@ from __future__ import annotations +from array import array import hashlib import json import os from pathlib import Path import subprocess import sys +from types import SimpleNamespace -from apps.benchmark.performance.baselines.timing_contracts import timing_contract -from tools.benchmark_qualification.catalog import discover, load_benchmark, select -from tools.benchmark_qualification.datasets import resolve_dataset -from tools.benchmark_qualification.runtime import ( +import pytest + +from tools import model_benchmark, prepare_coco_detection_dataset +from qualification_tests.benchmark_qualification.performance.references import hf_transformers, generic_reference +from qualification_tests.benchmark_qualification.performance.references.timing_contracts import timing_contract +from qualification_tests.benchmark_qualification import accuracy as qualification_accuracy +from qualification_tests.benchmark_qualification.performance import ( + qualification as qualification_performance, +) +from qualification_tests.benchmark_qualification import runtime as qualification_runtime +from qualification_tests.benchmark_qualification.catalog import ( + QualificationCase, + QualificationError, + discover, + load_benchmark, +) +from qualification_tests.benchmark_qualification.datasets import Dataset, resolve_dataset +from qualification_tests.benchmark_qualification.references import hf_encoder, hf_text_generation +from qualification_tests.benchmark_qualification.runtime import ( RuntimeContext, prepare_bundle, + reference_environment_options, + reference_python, run_command, write_model_descriptor, ) -REPOSITORY = Path(__file__).resolve().parents[2] +REPOSITORY = Path(__file__).resolve().parents[2] + + +def _example_case(tmp_path: Path, *, kind: str = "accuracy") -> QualificationCase: + return QualificationCase( + kind=kind, + model="example-model", + family="example", + name="parity", + benchmark="example_benchmark", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "example_task", + "precision": "fp32", + "build": {}, + }, + values={}, + source=tmp_path / "families/example/tests/benchmark/example-model.yaml", + reference_requirements=None, + ) + + +def test_qualification_yaml_rejects_duplicate_explicit_keys(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-model.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\nmodel: example-model\nmodel: replaced-model\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="duplicate key 'model'"): + discover(tmp_path) + + +def test_qualification_yaml_preserves_merge_keys(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-model.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example-model\n" + "candidate: {family: example, checkpoint: example/model, revision: " + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, task: example_task, " + "precision: fp32}\n" + "reference: &reference {runner: hf-transformers, mode: hf-eager}\n" + "accuracy: []\n" + "performance:\n" + " - name: parity\n" + " benchmark: example_performance\n" + " reference: {<<: *reference, precision: fp32}\n", + encoding="utf-8", + ) + + cases = discover(tmp_path) + + assert len(cases) == 1 + assert cases[0].values["reference"] == { + "runner": "hf-transformers", + "mode": "hf-eager", + "precision": "fp32", + } + + +def test_reference_environment_options_are_family_declared(tmp_path: Path, monkeypatch) -> None: + case = _example_case(tmp_path) + context = SimpleNamespace() + monkeypatch.setattr( + qualification_runtime, + "reference_environment_paths", + lambda *_args: {"reference_repo": "/reference/example"}, + ) + + assert reference_environment_options(case, context, {"variant": "official"}) == { + "variant": "official", + "reference_repo": "/reference/example", + } + with pytest.raises(QualificationError, match="duplicate.*reference_repo"): + reference_environment_options( + case, + context, + {"reference_repo": "/different/reference"}, + ) + + +def test_qualification_summary_is_written_as_report_json(tmp_path: Path) -> None: + summary = { + "schema_version": "trtmc.qualification-summary/v1", + "status": "passed", + "cases": [ + { + "model": "example-model", + "kind": "accuracy", + "case": "example-model/accuracy/parity", + "status": "passed", + } + ], + } + + model_benchmark._write_summary(tmp_path, summary) + + assert json.loads((tmp_path / "report.json").read_text(encoding="utf-8")) == summary + assert not (tmp_path / "summary.json").exists() + html_report = (tmp_path / "report.html").read_text(encoding="utf-8") + assert 'href="report.json"' in html_report + assert "summary.json" not in html_report + + +def test_family_configs_auto_discover_without_a_central_model_registry() -> None: + cases = discover(REPOSITORY) + assert cases + assert len({case.id for case in cases}) == len(cases) + for case in cases: + relative = case.source.relative_to(REPOSITORY / "families") + assert relative.parts[0] == case.family + assert relative.parts[1:3] == ("tests", "benchmark") + assert case.kind in {"accuracy", "performance"} + assert not any("l0" in case.model.lower() for case in cases) + + +def test_qualification_profiles_match_family_manifest_tasks_when_present() -> None: + profiles = {} + for case in discover(REPOSITORY): + manifest = case.source.parent.parent / "manifests" / f"{case.model}.json" + if ( + case.kind == "accuracy" + and case.benchmark == "imagenette_classification" + and manifest.is_file() + ): + profiles[case.model] = (case, manifest) + + assert profiles + for model, (case, manifest) in profiles.items(): + declared = json.loads(manifest.read_text(encoding="utf-8")) + assert case.candidate["task"] == declared["task"], model + + +def test_l0_configs_outside_the_benchmark_folder_are_not_discovered( + tmp_path: Path, +) -> None: + profile = tmp_path / "families/example/tests/l0/example.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + " revision: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + " task: text_generation\n" + " precision: fp16\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + + assert discover(tmp_path) == () + + +@pytest.mark.parametrize("revision", [None, "main", "abc1234"]) +def test_trusted_remote_code_requires_an_immutable_revision( + tmp_path: Path, revision: str | None +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + revision_line = "" if revision is None else f" revision: {revision}\n" + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + f"{revision_line}" + " task: text_generation\n" + " precision: fp16\n" + " trust_remote_code: true\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="immutable 40-character revision"): + discover(tmp_path) + + +@pytest.mark.parametrize("revision", [None, "main", "abc1234"]) +def test_remote_checkpoint_requires_an_immutable_revision( + tmp_path: Path, revision: str | None +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + revision_line = "" if revision is None else f" revision: {revision}\n" + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + f"{revision_line}" + " task: text_generation\n" + " precision: fp16\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="remote checkpoint requires an immutable"): + discover(tmp_path) + + +@pytest.mark.parametrize("model_directory", ["/tmp/model", "../model"]) +def test_candidate_model_directory_must_stay_in_the_family_environment( + tmp_path: Path, model_directory: str +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + f" model_directory: {model_directory}\n" + " task: classification\n" + " precision: fp16\n" + "accuracy: []\n" + "performance: []\n", + encoding="utf-8", + ) + + with pytest.raises(QualificationError, match="must stay inside the family environment"): + discover(tmp_path) + + +def test_one_model_file_owns_multiple_cases_without_testcase_indirection( + tmp_path: Path, +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-model.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example-model\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + " revision: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + " task: text_generation\n" + " precision: fp16\n" + "accuracy:\n" + " - name: continuation\n" + " benchmark: example_accuracy\n" + "performance:\n" + " - name: generate\n" + " benchmark: example_performance\n" + " operation: generate\n", + encoding="utf-8", + ) + + cases = discover(tmp_path) + + assert {case.kind for case in cases} == {"accuracy", "performance"} + assert {case.name for case in cases} == {"continuation", "generate"} + for case in cases: + assert case.source.parent.name == "benchmark" + assert "testcase" not in case.values + assert "testcase" not in str(case.values) + assert case.candidate["checkpoint"] == "example/model" + + +def test_family_environment_hook_is_discovered_with_the_model_profiles(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + profile.write_text( + "schema_version: trtmc.qualification/v1\n" + "model: example\n" + "candidate:\n" + " family: example\n" + " checkpoint: example/model\n" + " revision: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + " task: text_generation\n" + " precision: fp16\n" + "accuracy:\n" + " - name: continuation\n" + " benchmark: example_accuracy\n" + "performance: []\n", + encoding="utf-8", + ) + hook = profile.parent / "prepare_environment.py" + hook.write_text("# family hook\n", encoding="utf-8") + + (case,) = discover(tmp_path) + + assert case.environment_hook == hook.resolve() + + +def test_accuracy_forwards_declared_reference_model_load_options( + tmp_path: Path, + monkeypatch, +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example-model", + family="example", + name="continuation", + benchmark="mmlu_continuation", + candidate={ + "family": "example", + "checkpoint": "example/model", + "revision": "a" * 40, + "task": "text_generation", + "precision": "fp16", + "trust_remote_code": True, + "build": {"max_sequence_length": 16}, + }, + values={ + "samples": 1, + "prompt_token_limit": 8, + "truncation_side": "left", + "reference": { + "command": "reference.py", + "model": "example/reference-model", + "revision": "b" * 40, + "trust_remote_code": False, + "precision": "fp16", + "experts_implementation": "batched_mm", + }, + "request": {"max_new_tokens": 1, "temperature": 0.0}, + "gate": {"min_pass_rate": 1.0, "allowed_failures": 0}, + }, + source=profile, + reference_requirements=None, + ) + dataset_path = tmp_path / "mmlu.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "sample", "prompt": "Question"}]}), + encoding="utf-8", + ) + dataset = Dataset("mmlu-five-shot", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured: dict[str, object] = {} + captured_command: list[str] = [] + + def reference(command, *_args, **_kwargs): + captured_command.extend(command) + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + {"samples": [{"sample_id": "sample", "prompt": "Question", "token_ids": [1]}]} + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: { + "kind": "accuracy", + "selection": {"method": "first"}, + "metric": {"name": "exact_token_ids"}, + }, + ) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ([{"token_ids": [1]}], tmp_path / "model.bundle"), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert captured["model"] == "example/reference-model" + assert captured["revision"] == "b" * 40 + assert captured["trust_remote_code"] is False + assert captured["experts_implementation"] == "batched_mm" + assert runner in [Path(argument) for argument in captured_command] + + +def test_hf_accuracy_reference_uses_requested_expert_implementation(monkeypatch) -> None: + monkeypatch.setenv("TRTMC_QUALIFICATION_LOCAL_FILES_ONLY", "1") + + assert hf_text_generation._model_load_options( + {"revision": "revision", "experts_implementation": "batched_mm"} + ) == { + "revision": "revision", + "local_files_only": True, + "experts_implementation": "batched_mm", + } + assert hf_text_generation._model_load_options({"trust_remote_code": True}) == { + "local_files_only": True, + "trust_remote_code": True, + } + assert hf_text_generation._precision_load_options("fp16") == {"torch_dtype": "fp16"} + + +def test_image_classification_accuracy_compares_top1_and_gold_accuracy( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "Imagenette" + image_root = dataset_root / "images" + image_root.mkdir(parents=True) + for name in ("a.jpeg", "b.jpeg"): + (image_root / name).write_bytes(b"fixture") + dataset_path = dataset_root / "manifest.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + {"id": "a", "image": "images/a.jpeg", "label": 1}, + {"id": "b", "image": "images/b.jpeg", "label": 3}, + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + case = QualificationCase( + kind="accuracy", + model="example", + family="example", + name="imagenette-parity", + benchmark="imagenette_classification", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "classification", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 2, + "reference": {"precision": "fp32", "batch_size": 2}, + "gate": { + "min_top1_agreement": 1.0, + "max_top1_accuracy_drop_from_hf": 0.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("imagenette-validation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured: dict[str, object] = {} + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "a", "top_class": 1}, + {"sample_id": "b", "top_class": 2}, + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "image_classification_top1_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"top_class": 1}, {"top_class": 2}], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"] == { + "samples": 2, + "top1_agreement": 1.0, + "reference_top1_accuracy": 0.5, + "candidate_top1_accuracy": 0.5, + "top1_accuracy_drop_from_hf": 0.0, + } + assert captured["batch_size"] == 2 + assert captured["samples"][0]["image_path"] == str(image_root / "a.jpeg") + + +def test_performance_resolves_profile_owned_relative_assets(tmp_path: Path) -> None: + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + image = profile.parent.parent / "data/test.jpeg" + image.parent.mkdir(parents=True) + image.write_bytes(b"fixture") + case = QualificationCase( + kind="performance", + model="example", + family="example", + name="classify", + benchmark="image_classification_performance", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "classification", + "precision": "fp16", + "build": {}, + }, + values={}, + source=profile, + reference_requirements=None, + ) + + resolved = qualification_performance._resolve_family_assets( + case, {"image_path": "../data/test.jpeg", "batch_size": 1} + ) + + assert resolved == {"image_path": str(image.resolve()), "batch_size": 1} + + +def test_robot_action_accuracy_compares_complete_action_chunk(tmp_path: Path, monkeypatch) -> None: + data = tmp_path / "families/example/tests/data" + data.mkdir(parents=True) + image = data / "image.png" + state = data / "state.f32" + image.write_bytes(b"fixture") + state.write_bytes(b"state") + dataset_path = data / "requests.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "recorded", "image": image.name, "state": state.name}]}), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example-control", + family="example", + name="recorded-action-parity", + benchmark="robot_action_parity", + candidate={ + "family": "example", + "checkpoint": "example/control", + "task": "robot_control", + "precision": "fp32", + "build": {}, + }, + values={ + "samples": 1, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "action_max_abs_error": 0.00005, + "action_mean_abs_error": 0.000005, + "action_rmse": 0.00001, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("robot-action-recorded-observation", dataset_path, "family", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + reference = { + "action_steps": 2, + "action_dim": 2, + "action_values": 4, + "actions": [0.1, -0.2, 0.3, -0.4], + "finite": True, + } + candidate = { + **reference, + "within_training_bounds": True, + } + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "robot_action_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, + "_family_image_reference", + lambda *_args: [reference], + ) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ([candidate], tmp_path / "act.bundle"), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["sample_pass_rate"] == 1.0 + assert result["metrics"]["action_max_abs_error"] == pytest.approx(0.0) + + +def test_coco_object_detection_accuracy_uses_ground_truth_map(tmp_path: Path, monkeypatch) -> None: + dataset_root = tmp_path / "COCO2017_object_detection" + dataset_root.mkdir() + image = dataset_root / "image.jpeg" + image.write_bytes(b"fixture") + dataset_path = dataset_root / "manifest.json" + dataset_path.write_text( + json.dumps( + { + "sampling": "test fixture", + "requests": [ + { + "id": "image", + "image": "image.jpeg", + "annotations": [ + { + "bbox_xyxy": [10.0, 20.0, 50.0, 70.0], + "category_id": 3, + "category_index": 2, + } + ], + } + ], + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/detr/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="detr", + name="coco2017-object-detection", + benchmark="coco2017_object_detection", + candidate={ + "family": "detr", + "checkpoint": "facebook/example", + "task": "object_detection", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "label_space": "coco-category-id", + "request": {"score_threshold": 0.5}, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "max_map_50_95_drop": 0.02, + "max_map_50_drop": 0.02, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("coco2017-object-detection", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + + def reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "image", + "boxes": [[10.0, 20.0, 50.0, 70.0]], + "scores": [0.9], + "class_ids": [3], + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "coco_object_detection_accuracy"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [ + { + "boxes": [10.0, 20.0, 50.0, 70.0], + "scores": [0.89], + "class_ids": [3], + } + ], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["candidate_map_50_95"] == pytest.approx(1.0) + assert result["metrics"]["reference_map_50_95"] == pytest.approx(1.0) + assert result["metrics"]["map_50_95_drop"] == pytest.approx(0.0) + + +def test_prepare_coco_detection_dataset_is_balanced_and_keeps_real_boxes( + tmp_path: Path, +) -> None: + coco_root = tmp_path / "source" + (coco_root / "annotations").mkdir(parents=True) + (coco_root / "val2017").mkdir() + images = [ + {"id": 11, "file_name": "000000000011.jpg", "width": 100, "height": 100}, + {"id": 22, "file_name": "000000000022.jpg", "width": 100, "height": 100}, + ] + annotations = [ + { + "id": 101, + "image_id": 11, + "category_id": 1, + "bbox": [10, 20, 30, 40], + "area": 1200, + "iscrowd": 0, + }, + { + "id": 202, + "image_id": 22, + "category_id": 3, + "bbox": [1, 2, 10, 20], + "area": 200, + "iscrowd": 0, + }, + ] + categories = [{"id": 1, "name": "person"}, {"id": 3, "name": "car"}] + (coco_root / "annotations" / "instances_val2017.json").write_text( + json.dumps({"images": images, "annotations": annotations, "categories": categories}), + encoding="utf-8", + ) + for image in images: + (coco_root / "val2017" / image["file_name"]).write_bytes(b"fixture") + + output = prepare_coco_detection_dataset.prepare(coco_root, tmp_path / "output", 2) + manifest = json.loads(output.read_text(encoding="utf-8")) + + assert [request["image_id"] for request in manifest["requests"]] == [11, 22] + assert manifest["requests"][0]["annotations"] == [ + { + "id": 101, + "category_id": 1, + "category_index": 0, + "bbox_xyxy": [10.0, 20.0, 40.0, 60.0], + "area": 1200.0, + } + ] + assert manifest["requests"][1]["annotations"][0]["category_index"] == 1 + + +def test_prompted_segmentation_masks_match_independent_of_order() -> None: + first = { + "num_masks": 2, + "height": 2, + "width": 2, + "mask_kind": "logits", + "masks": [1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0], + } + second = { + "num_masks": 2, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [0, 1, 1, 0, 1, 0, 0, 1], + } + + _, _, candidate = qualification_accuracy._binary_masks(first, "candidate") + _, _, reference = qualification_accuracy._binary_masks(second, "reference") + + assert qualification_accuracy._match_masks(candidate, reference) == [1.0, 1.0] + + +def test_text_prompted_instances_match_masks_boxes_and_scores() -> None: + summary = { + "num_masks": 1, + "height": 2, + "width": 2, + "mask_kind": "binary", + "masks": [1, 0, 0, 1], + "iou_scores": [0.9], + "boxes": [[0.0, 0.0, 2.0, 2.0]], + "box_coordinates": "original_image_pixels_xyxy", + } + parsed = qualification_accuracy._instance_masks(summary, "test") + matches = qualification_accuracy._match_instances(parsed, parsed) + + assert matches == [{"mask_iou": 1.0, "box_iou": 1.0, "score_abs_error": 0.0}] + + +def test_vision_language_text_distance_is_normalized() -> None: + left = qualification_accuracy._normalized_answer(" Red\ncar ") + right = qualification_accuracy._normalized_answer("red car") + + assert qualification_accuracy._normalized_edit_distance(left, right) == 0.0 + assert qualification_accuracy._normalized_edit_distance(left, "blue car") > 0.15 + + +def test_ocr_accuracy_uses_dataset_questions_and_reports_gold_matches( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "OCRBench_v2/unified" + image_root = dataset_root / "images" + image_root.mkdir(parents=True) + for name in ("a.jpg", "b.jpg"): + (image_root / name).write_bytes(b"fixture") + dataset_path = dataset_root / "dataset.json" + dataset_path.write_text( + json.dumps( + { + "samples": [ + { + "id": "a", + "question": "What is the setting?", + "media": [{"type": "image", "path": "images/a.jpg"}], + "answer": {"primary": "enabled", "aliases": ["enabled", "on"]}, + }, + { + "id": "b", + "question": "Which application?", + "media": [{"type": "image", "path": "images/b.jpg"}], + "answer": {"primary": "Facebook", "aliases": ["Facebook"]}, + }, + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example-ocr.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example-ocr", + family="example", + name="ocrbench-v2-parity", + benchmark="ocrbench_v2_parity", + candidate={ + "family": "example", + "checkpoint": "example/ocr", + "revision": "a" * 40, + "task": "vision_language_generation", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 2, + "request": {"max_new_tokens": 16}, + "reference": {"command": "reference.py", "precision": "bf16"}, + "gate": {"max_normalized_edit_distance": 0.1, "min_sample_pass_rate": 1.0}, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("ocrbench-v2", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path, + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured_reference: dict[str, object] = {} + captured_candidate: list[dict[str, object]] = [] + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured_reference.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "a", "text": "enabled"}, + {"sample_id": "b", "text": "Facebook"}, + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def candidate(_case, _context, _output, _operation, requests): + captured_candidate.extend(requests) + return ([{"text": "enabled"}, {"text": "facebook"}], tmp_path / "model.bundle") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "ocr_text_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr(qualification_accuracy, "_candidate_outputs", candidate) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["candidate_normalized_gold_match_rate"] == 1.0 + assert result["metrics"]["reference_normalized_gold_match_rate"] == 1.0 + assert captured_reference["samples"][0]["prompt"] == "What is the setting?" + assert captured_candidate[1]["request"]["prompt"] == "Which application?" + + +def test_localization_accuracy_matches_unordered_boxes_and_points() -> None: + first_kind, first_boxes = qualification_accuracy._localization_values( + "cars<10><20><100><200><300><400><500><600>" + ) + second_kind, second_boxes = qualification_accuracy._localization_values( + "cars<300><400><500><600><10><20><100><200>" + ) + first_point_kind, first_points = qualification_accuracy._localization_values( + "cars<100><200>" + ) + second_point_kind, second_points = qualification_accuracy._localization_values( + "cars<106><208>" + ) + + assert first_kind == second_kind == "box" + assert qualification_accuracy._localization_alignment(first_boxes, second_boxes, "box") == 1.0 + assert first_point_kind == second_point_kind == "point" + assert ( + qualification_accuracy._localization_alignment(first_points, second_points, "point") == 10.0 + ) + with pytest.raises(QualificationError, match="reference tag"): + qualification_accuracy._localization_values("<10><20><100><200>") + + +def test_reranking_order_is_stable_and_score_validation_is_strict() -> None: + assert qualification_accuracy._reranking_order([0.7, 0.2, 0.7]) == [0, 2, 1] + assert qualification_accuracy._reranking_scores({"scores": [0.7, -0.2]}, "test") == [ + 0.7, + -0.2, + ] + with pytest.raises(QualificationError, match="non-finite"): + qualification_accuracy._reranking_scores({"scores": [float("nan")]}, "test") + + +def test_semantic_segmentation_accuracy_compares_pixel_and_class_iou( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "Imagenette" + dataset_root.mkdir() + image = dataset_root / "image.jpeg" + image.write_bytes(b"fixture") + dataset_path = dataset_root / "manifest.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="example", + name="segmentation-parity", + benchmark="imagenette_segmentation_parity", + candidate={ + "family": "example", + "checkpoint": "example/segmentation", + "task": "segmentation", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "min_pixel_accuracy": 0.75, + "min_mean_iou": 0.58, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("imagenette-validation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + + def reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "image", + "height": 2, + "width": 2, + "mask": [0, 0, 1, 1], + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "semantic_segmentation_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"height": 2, "width": 2, "mask": [0, 0, 1, 0]}], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["mean_pixel_accuracy"] == 0.75 + assert result["metrics"]["mean_iou"] >= 0.58 + + +def test_image_feature_accuracy_compares_vectors_and_knn_utility( + tmp_path: Path, monkeypatch +) -> None: + dataset_root = tmp_path / "beans" + (dataset_root / "images/train").mkdir(parents=True) + (dataset_root / "images/test").mkdir(parents=True) + (dataset_root / "queries").mkdir() + for split in ("train", "test"): + for index in range(2): + (dataset_root / f"images/{split}/{index}.jpg").write_bytes(b"fixture") + (dataset_root / "bank.json").write_text( + json.dumps( + { + "samples": [ + {"image": "images/train/0.jpg", "label": 0, "source_index": 0}, + {"image": "images/train/1.jpg", "label": 1, "source_index": 1}, + ] + } + ), + encoding="utf-8", + ) + (dataset_root / "queries/test.json").write_text( + json.dumps( + { + "samples": [ + {"image": "../images/test/0.jpg", "label": 0, "source_index": 0}, + {"image": "../images/test/1.jpg", "label": 1, "source_index": 1}, + ] + } + ), + encoding="utf-8", + ) + dataset_path = dataset_root / "dataset.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "inputs": { + "bank_manifest": "bank.json", + "query_manifest": "queries/test.json", + } + } + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="example", + name="beans-knn-parity", + benchmark="beans_image_feature_knn", + candidate={ + "family": "example", + "checkpoint": "example/features", + "task": "image_features", + "precision": "fp16", + "build": {"max_sequence_length": 1}, + }, + values={ + "bank_samples_per_class": 1, + "query_samples_per_class": 1, + "knn_k": 1, + "knn_temperature": 0.07, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "min_pooler_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "min_knn_top1_agreement": 1.0, + "max_knn_accuracy_drop_from_hf": 0.0, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("image-feature-knn-v1", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + vectors = [[1.0, 0.0], [0.0, 1.0], [0.9, 0.1], [0.1, 0.9]] + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + samples = json.loads(request.read_text(encoding="utf-8"))["samples"] + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": sample["sample_id"], "pooler_output": vector} + for sample, vector in zip(samples, vectors, strict=True) + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "image_feature_knn_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"pooler_output": vector} for vector in vectors], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["vector_pass_rate"] == 1.0 + assert result["metrics"]["knn_top1_agreement"] == 1.0 + assert result["metrics"]["candidate_knn_top1_accuracy"] == 1.0 + + +def test_timm_classification_uses_a_generic_reference_adapter() -> None: + assert "timm-classification" in generic_reference.ADAPTERS + assert generic_reference.LOADERS["timm-classification"] is generic_reference._load_vision + + +def test_glm_asr_performance_uses_its_family_reference() -> None: + case = next( + value + for value in discover(REPOSITORY) + if value.model == "glmasr-nano-fp16" and value.kind == "performance" + ) + assert case.values["reference"]["script"] == "tests/benchmark/reference.py" + assert "hf-chat-asr" not in generic_reference.ADAPTERS + + +def test_speech_accuracy_reports_reference_and_labeled_wer(tmp_path: Path, monkeypatch) -> None: + data_root = tmp_path / "datasets" + audio_root = data_root / "speech/audio" + audio_root.mkdir(parents=True) + for name in ("a.flac", "b.flac"): + (audio_root / name).write_bytes(b"fixture") + dataset_path = data_root / "speech/dataset.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "id": "a", + "messages": [ + { + "role": "user", + "content": [{"type": "audio", "audio": "speech/audio/a.flac"}], + } + ], + "reference": "hello world", + }, + { + "id": "b", + "messages": [ + { + "role": "user", + "content": [{"type": "audio", "audio": "speech/audio/b.flac"}], + } + ], + "reference": "one two", + }, + ] + } + ), + encoding="utf-8", + ) + profile = tmp_path / "families/example/tests/benchmark/example.yaml" + profile.parent.mkdir(parents=True) + runner = profile.parent / "reference.py" + runner.write_text("# family reference\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="example", + family="example", + name="librispeech-wer", + benchmark="librispeech_transcription", + candidate={ + "family": "example", + "checkpoint": "org/example", + "task": "transcription", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 2, + "request": {"max_new_tokens": 32}, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": { + "max_wer_to_reference": 0.3, + "max_wer_increase_from_reference": 0.3, + }, + }, + source=profile, + reference_requirements=None, + ) + dataset = Dataset("speech", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=data_root, + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + + def reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + wavs = [tmp_path / "a.wav", tmp_path / "b.wav"] + for wav in wavs: + wav.write_bytes(b"fixture") + output.write_text( + json.dumps( + { + "samples": [ + {"sample_id": "a", "text": "hello world", "audio_path": str(wavs[0])}, + {"sample_id": "b", "text": "one two", "audio_path": str(wavs[1])}, + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "speech_transcription_wer_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [{"text": "hello world"}, {"text": "one three"}], + tmp_path / "example.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"] == { + "samples": 2, + "reference_wer": 0.0, + "candidate_wer": 0.25, + "wer_increase_from_reference": 0.25, + "wer_to_reference": 0.25, + } + + +def test_accuracy_forwards_seq2seq_reference_contract_and_nested_dataset_input( + tmp_path: Path, monkeypatch +) -> None: + dataset_path = tmp_path / "translation.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "sample_id": "translation-0", + "inputs": {"prompt": "The house is wonderful."}, + } + ] + } + ), + encoding="utf-8", + ) + case = QualificationCase( + kind="accuracy", + model="translation-model", + family="translation", + name="translation-parity", + benchmark="translation_parity", + candidate={ + "family": "translation", + "checkpoint": "org/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "prompt_token_limit": 96, + "reference": { + "precision": "fp16", + "task": "seq2seq-lm", + "output_token_policy": "strip-start-and-eos", + "source_language_placement": "replace-final-unk", + }, + "request": { + "max_new_tokens": 128, + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + }, + "gate": {"min_pass_rate": 1.0, "allowed_failures": 0}, + }, + source=tmp_path / "translation.yaml", + reference_requirements=None, + ) + dataset = Dataset("translation", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured_reference: dict[str, object] = {} + captured_candidate: list[dict[str, object]] = [] + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured_reference.update(json.loads(request.read_text(encoding="utf-8"))) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "translation-0", + "prompt": "The house is wonderful.", + "token_ids": [4, 5], + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def candidate(_case, _context, _output, _operation, requests): + captured_candidate.extend(requests) + return ([{"token_ids": [4, 5]}], tmp_path / "translation.bundle") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "exact_token_ids"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr(qualification_accuracy, "_candidate_outputs", candidate) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert captured_reference["task"] == "seq2seq-lm" + assert captured_reference["output_token_policy"] == "strip-start-and-eos" + assert captured_reference["generation"]["source_language_placement"] == ("replace-final-unk") + assert captured_reference["samples"] == [ + {"sample_id": "translation-0", "prompt": "The house is wonderful."} + ] + assert captured_candidate[0]["request"]["forced_bos_token_id"] == 256057 + + +def test_hf_accuracy_reference_selects_the_seq2seq_model_class() -> None: + causal = object() + seq2seq = object() + + assert hf_text_generation._model_class("causal-lm", causal, seq2seq) is causal + assert hf_text_generation._model_class("seq2seq-lm", causal, seq2seq) is seq2seq + with pytest.raises(ValueError, match="unsupported reference task"): + hf_text_generation._model_class("encoder", causal, seq2seq) + + +def test_hf_accuracy_reference_applies_explicit_translation_languages() -> None: + class Tokenizer: + src_lang = None + unk_token_id = 0 + + @staticmethod + def convert_tokens_to_ids(value: str) -> int: + return {"eng_Latn": 256047, "fra_Latn": 256057}.get(value, 0) + + @staticmethod + def convert_ids_to_tokens(value: int) -> str: + return {256047: "eng_Latn", 256057: "fra_Latn"}[value] + + tokenizer = Tokenizer() + controls, source_token_id = hf_text_generation._translation_controls( + tokenizer, + { + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + }, + ) + + assert tokenizer.src_lang == "eng_Latn" + assert controls == {"forced_bos_token_id": 256057} + assert source_token_id is None + + +@pytest.mark.parametrize("runner", [hf_text_generation, hf_transformers]) +def test_hf_translation_supports_replace_final_unknown_tokenizer(runner) -> None: + import torch + + class GenericTranslationTokenizer: + unk_token_id = 3 + + @staticmethod + def convert_tokens_to_ids(value: str) -> int: + return {"eng_Latn": 256047, "fra_Latn": 256057}.get(value, 3) + + @staticmethod + def convert_ids_to_tokens(value: int) -> str: + return {256047: "eng_Latn", 256057: "fra_Latn"}[value] + + tokenizer = GenericTranslationTokenizer() + request = { + "source_language": "eng_Latn", + "source_language_token_id": 256047, + "target_language": "fra_Latn", + "forced_bos_token_id": 256057, + } + with pytest.raises(ValueError, match="source_language_placement"): + runner._translation_controls(tokenizer, request) + + controls, source_token_id = runner._translation_controls( + tokenizer, + { + **request, + "source_language_placement": "replace-final-unk", + }, + ) + encoded = { + "input_ids": torch.tensor([[17, 2, 3]]), + "attention_mask": torch.tensor([[1, 1, 1]]), + } + + runner._apply_source_language(encoded, source_token_id, tokenizer) + + assert controls == {"forced_bos_token_id": 256057} + assert source_token_id == 256047 + assert encoded["input_ids"].tolist() == [[17, 2, 256047]] + + +@pytest.mark.parametrize( + ("row", "expected_status", "expected_error"), + [ + ( + { + "status": "contract-mismatch", + "comparison": {"reason": "generated token ids differ"}, + "reference_attempts": [], + }, + "failed", + None, + ), + ( + {"status": "white", "error": "candidate command failed"}, + "error", + "candidate command failed", + ), + ( + { + "status": "white", + "error": "reference command and configured fallback failed", + "reference_attempts": [ + { + "mode": "torch-compile", + "exit_code": 1, + "fallback_reason": "reference command failed", + } + ], + }, + "error", + "reference command and configured fallback failed", + ), + ], +) +def test_performance_distinguishes_model_contract_failures_from_execution_errors( + tmp_path: Path, + monkeypatch, + row: dict[str, object], + expected_status: str, + expected_error: str | None, +) -> None: + case = QualificationCase( + kind="performance", + model="example-model", + family="example", + name="generate", + benchmark="text_generation_performance", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={ + "operation": "generate", + "request": {"prompt": "Hello", "max_new_tokens": 1}, + "measurement": {"warmup": 1, "iterations": 10}, + "reference": { + "runner": "hf-transformers", + "mode": "hf-eager", + "precision": "fp32", + "output_contract": "exact-token-ids", + }, + }, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=tmp_path / "runtime", + trtmc_bench=tmp_path / "trtmc-bench", + worker=tmp_path / "worker", + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + definition = { + "reference_timing": { + "timing_scope": "public_operation_call_wall", + "input_preparation_included": True, + "asset_loading_included": False, + }, + "stability": { + "samples": 10, + "max_half_median_change_percent": 5.0, + "median_band_percent": 5.0, + "minimum_samples_within_band": 8, + "retries": 1, + }, + } + + def run_single_case(*_args, **_kwargs): + return {"id": "qualification.example.generate", **row} + + monkeypatch.setattr(qualification_performance, "load_benchmark", lambda *_: definition) + monkeypatch.setattr( + qualification_performance, + "require_candidate", + lambda *_: (context.worker, context.runtime_root), + ) + monkeypatch.setattr( + qualification_performance, "reference_python", lambda *_: Path(sys.executable) + ) + monkeypatch.setattr(qualification_performance, "run_case", run_single_case) + + result = qualification_performance.run_performance(case, context) + + assert result["status"] == expected_status + assert result.get("error") == expected_error + + +def test_hf_accuracy_reference_normalizes_seq2seq_control_tokens() -> None: + assert hf_text_generation._normalize_seq2seq_tokens( + [2, 256057, 1034, 248075, 2], + decoder_start_token_id=2, + eos_token_id=2, + policy="strip-start-and-eos", + ) == [256057, 1034, 248075] + assert hf_text_generation._normalize_seq2seq_tokens( + [0, 17, 1], + decoder_start_token_id=0, + eos_token_id=1, + policy="strip-start", + ) == [17, 1] + + +def test_sts_samples_expand_pairs_with_family_owned_prompt_prefix(tmp_path: Path) -> None: + dataset = tmp_path / "sts.jsonl" + dataset.write_text( + "\n".join( + ( + json.dumps( + { + "genre": "captions", + "score": 2.5, + "sentence1": "A girl is styling her hair.", + "sentence2": "A girl is brushing her hair.", + } + ), + json.dumps( + { + "genre": "news", + "score": 4.0, + "sentence1": "One sentence.", + "sentence2": "Another sentence.", + } + ), + ) + ) + + "\n", + encoding="utf-8", + ) + + assert qualification_accuracy._sts_samples(dataset, 1, "query: ") == [ + { + "sample_id": "stsbenchmark-000000-a", + "pair_id": "stsbenchmark-000000", + "pair_side": "sentence1", + "score": 2.5, + "prompt": "query: A girl is styling her hair.", + }, + { + "sample_id": "stsbenchmark-000000-b", + "pair_id": "stsbenchmark-000000", + "pair_side": "sentence2", + "score": 2.5, + "prompt": "query: A girl is brushing her hair.", + }, + ] + + +def test_encoder_embedding_comparison_restores_pre_refactor_gates() -> None: + reference = [ + { + "sample_id": "pair-a", + "pair_id": "pair", + "pair_side": "sentence1", + "score": 5.0, + "vector": [1.0, 0.0], + }, + { + "sample_id": "pair-b", + "pair_id": "pair", + "pair_side": "sentence2", + "score": 5.0, + "vector": [0.8, 0.6], + }, + ] + candidate = [{"values": [1.0, 0.0]}, {"values": [0.8, 0.6]}] + + result = qualification_accuracy._compare_encoder_embeddings( + reference, + candidate, + { + "min_vector_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "max_pair_cosine_abs_delta": 0.02, + }, + ) + + assert result["status"] == "passed" + assert result["metrics"]["vector_pass_rate"] == 1.0 + assert result["metrics"]["max_pair_cosine_abs_delta"] == pytest.approx(0.0) + assert result["metrics"]["hf_sts_spearman"] is None + assert result["metrics"]["candidate_sts_spearman"] is None + + +@pytest.mark.parametrize( + ("task", "expected_mode", "expected_operation"), + [("encoding", "cls", "encode"), ("embedding", "embedding", "embed")], +) +def test_encoder_accuracy_uses_task_semantics_without_model_specific_runner( + tmp_path: Path, + monkeypatch, + task: str, + expected_mode: str, + expected_operation: str, +) -> None: + dataset_path = tmp_path / "sts.jsonl" + dataset_path.write_text( + json.dumps( + { + "genre": "captions", + "score": 3.0, + "sentence1": "Sentence one.", + "sentence2": "Sentence two.", + } + ) + + "\n", + encoding="utf-8", + ) + case = QualificationCase( + kind="accuracy", + model="encoder-model", + family="encoder", + name="stsbenchmark-parity", + benchmark="stsbenchmark_embedding_parity", + candidate={ + "family": "encoder", + "checkpoint": "org/model", + "task": task, + "precision": "fp16", + "build": {"max_sequence_length": 128}, + }, + values={ + "samples": 1, + "reference": {"precision": "fp32"}, + "gate": { + "min_vector_cosine": 0.999, + "min_vector_pass_rate": 1.0, + "max_pair_cosine_abs_delta": 0.02, + }, + }, + source=tmp_path / "encoder.yaml", + reference_requirements=None, + ) + dataset = Dataset("stsbenchmark-test", dataset_path, "provided", "digest") + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + captured_reference: dict[str, object] = {} + captured_candidate: dict[str, object] = {} + + def reference(command, *_args, **_kwargs): + request = Path(command[command.index("--request") + 1]) + output = Path(command[command.index("--output") + 1]) + captured_reference.update(json.loads(request.read_text(encoding="utf-8"))) + samples = captured_reference["samples"] + output.write_text( + json.dumps( + { + "samples": [ + {**sample, "vector": [1.0, float(index)]} + for index, sample in enumerate(samples) + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + def candidate(_case, _context, _output, operation, requests): + captured_candidate.update({"operation": operation, "requests": requests}) + return ( + [{"values": [1.0, float(index)]} for index, _request in enumerate(requests)], + tmp_path / "encoder.bundle", + ) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", reference) + monkeypatch.setattr(qualification_accuracy, "_candidate_outputs", candidate) -def test_family_configs_auto_discover_without_a_central_model_registry() -> None: - cases = discover(REPOSITORY) + result = qualification_accuracy._encoder_embedding_parity( + case, context, dataset, tmp_path / "output" + ) - assert cases - assert len({case.id for case in cases}) == len(cases) - for case in cases: - relative = case.source.relative_to(REPOSITORY / "families") - assert relative.parts[0] == case.family - assert relative.parts[1:3] == ("tests", "benchmark") - assert case.kind in {"accuracy", "performance"} - assert not any("l0" in case.model.lower() for case in cases) + assert result["status"] == "passed" + assert captured_reference["mode"] == expected_mode + assert captured_candidate["operation"] == expected_operation + assert [request["request"] for request in captured_candidate["requests"]] == [ + {"prompt": "Sentence one."}, + {"prompt": "Sentence two."}, + ] -def test_one_model_file_owns_multiple_cases_without_testcase_indirection() -> None: - cases = select(discover(REPOSITORY), ["gpt2-125m"]) +def test_hf_encoder_rejects_unknown_vector_mode() -> None: + with pytest.raises(ValueError, match="unsupported encoder vector mode"): + hf_encoder._vector_mode("mean") - assert {case.kind for case in cases} == {"accuracy", "performance"} - assert {case.name for case in cases} == {"mmlu-continuation", "generate-64"} - for case in cases: - assert case.source.parent.name == "benchmark" - assert "testcase" not in case.values - assert "testcase" not in str(case.values) - assert case.candidate["checkpoint"] == "openai-community/gpt2" + +def test_hf_encoder_resolves_family_declared_transformers_classes() -> None: + auto_model = object() + auto_tokenizer = object() + custom_model = object() + custom_tokenizer = object() + transformers = SimpleNamespace( + AutoModel=auto_model, + AutoTokenizer=auto_tokenizer, + CustomEncoder=custom_model, + CustomTokenizer=custom_tokenizer, + ) + + assert hf_encoder._reference_classes(transformers, "auto", "auto") == ( + auto_model, + auto_tokenizer, + ) + assert hf_encoder._reference_classes( + transformers, + "transformers.CustomEncoder", + "transformers.CustomTokenizer", + ) == (custom_model, custom_tokenizer) + with pytest.raises(ValueError, match="unsupported Transformers class"): + hf_encoder._reference_classes( + transformers, + "dpr-context-encoder", + "transformers.CustomTokenizer", + ) def test_shared_definitions_own_dataset_and_metric_not_models() -> None: @@ -55,8 +2034,59 @@ def test_shared_definitions_own_dataset_and_metric_not_models() -> None: assert "models" not in definition assert definition["kind"] == case.kind if case.kind == "accuracy": - assert definition["dataset"]["id"] assert definition["metric"]["name"] + if definition["metric"]["name"] == "task_output_parity": + assert definition["input"]["adapter"] == "configured_request" + assert "dataset" not in definition + else: + assert definition["dataset"]["id"] + + +def test_fixed_dataset_selection_uses_declared_indices() -> None: + rows = [f"sample-{index}" for index in range(8)] + + selected, receipt = qualification_accuracy._select_dataset_rows( + rows, + {"selection": {"method": "fixed-indices", "indices": [6, 2, 4]}}, + 2, + "example", + ) + + assert selected == ["sample-6", "sample-2"] + assert receipt == {"method": "fixed-indices", "indices": [6, 2]} + + +@pytest.mark.parametrize( + ("indices", "message"), + [ + ([1, 1], "must be unique"), + ([1, True], "nonnegative integers"), + ([1, 8], "out of range"), + ], +) +def test_fixed_dataset_selection_rejects_invalid_indices(indices: list[int], message: str) -> None: + with pytest.raises(QualificationError, match=message): + qualification_accuracy._select_dataset_rows( + list(range(8)), + {"selection": {"method": "fixed-indices", "indices": indices}}, + 2, + "example", + ) + + +@pytest.mark.parametrize( + "benchmark", + ["mmlu_continuation", "librispeech_transcription", "ocrbench_v2_parity"], +) +def test_slow_dataset_benchmarks_keep_ten_fixed_representative_samples( + benchmark: str, +) -> None: + case = next(case for case in discover(REPOSITORY) if case.benchmark == benchmark) + selection = load_benchmark(REPOSITORY, case)["selection"] + + assert selection["method"] == "fixed-indices" + assert len(selection["indices"]) == 10 + assert len(set(selection["indices"])) == 10 def test_shared_performance_definitions_own_complete_reference_timing() -> None: @@ -71,20 +2101,13 @@ def test_shared_performance_definitions_own_complete_reference_timing() -> None: "input_preparation_included", "asset_loading_included", } - assert timing_contract( - runner=str(case.values["reference"]["runner"]), declared=declared - ) + assert timing_contract(runner=str(case.values["reference"]["runner"]), declared=declared) def test_internal_automation_is_separate_from_the_installed_benchmark() -> None: old_application = REPOSITORY / "apps/benchmark/qualification" assert list(old_application.rglob("*.py")) == [] - internal_sources = [ - path.read_text(encoding="utf-8") - for path in (REPOSITORY / "tools/benchmark_qualification").rglob("*.py") - ] - assert not any("ManifestCatalog" in source for source in internal_sources) - assert not any("tests/manifests" in source for source in internal_sources) + assert list((REPOSITORY / "tools/benchmark_qualification").rglob("*.py")) == [] user_sources = [ path.read_text(encoding="utf-8") for path in (REPOSITORY / "apps/benchmark").rglob("*.py") @@ -94,15 +2117,244 @@ def test_internal_automation_is_separate_from_the_installed_benchmark() -> None: def test_candidate_descriptor_is_generated_from_public_build_inputs(tmp_path: Path) -> None: - case = select(discover(REPOSITORY), ["gpt2-125m"])[0] + case = QualificationCase( + kind="accuracy", + model="example-model", + family="example", + name="continuation", + benchmark="example_accuracy", + candidate={ + "family": "example", + "checkpoint": "example/model", + "revision": "a" * 40, + "task": "text_generation", + "precision": "fp16", + "trust_remote_code": True, + "build": {"max_sequence_length": 1024}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) descriptor = write_model_descriptor(case, tmp_path, {"prompt": "hello"}) - value = descriptor.read_text(encoding="utf-8") + value = json.loads(descriptor.read_text(encoding="utf-8")) + + assert value["hf_id"] == "example/model" + assert value["hf_revision"] == "a" * 40 + assert value["task"] == "text_generation" + assert value["max_sequence_length"] == 1024 + assert value["trust_remote_code"] is True + assert "tests/manifests" not in descriptor.read_text(encoding="utf-8") + + +def test_candidate_descriptor_preserves_family_selected_task(tmp_path: Path) -> None: + case = QualificationCase( + kind="performance", + model="geometry-model", + family="geometry", + name="geometry", + benchmark="geometry_performance", + candidate={ + "family": "geometry", + "checkpoint": "example/geometry", + "task": "legacy_geometry", + "selected_task": "image_to_metric_geometry", + "precision": "fp32", + "build": {}, + }, + values={}, + source=tmp_path / "geometry.yaml", + reference_requirements=None, + ) + + descriptor = write_model_descriptor(case, tmp_path, {"image_path": "/data/image.png"}) + testcase = json.loads(descriptor.read_text(encoding="utf-8"))["testcases"][0] + + assert testcase["selected_task"] == "image_to_metric_geometry" + assert testcase["image_path"] == "/data/image.png" + + +def test_candidate_descriptor_uses_family_environment_model_directory( + tmp_path: Path, monkeypatch +) -> None: + environment = tmp_path / "family-environment" + python = environment / "bin/python" + python.parent.mkdir(parents=True) + python.write_text("", encoding="utf-8") + model_directory = environment / "prepared/model" + model_directory.mkdir(parents=True) + case = QualificationCase( + kind="accuracy", + model="source-model", + family="source_family", + name="parity", + benchmark="source_parity", + candidate={ + "family": "source_family", + "checkpoint": "example/source-model", + "model_directory": "prepared/model", + "task": "classification", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "source-model.yaml", + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + monkeypatch.setattr( + "qualification_tests.benchmark_qualification.runtime.reference_python", lambda *_args: python + ) + + descriptor = write_model_descriptor( + case, tmp_path, {"image_path": "/data/image.png"}, context=context + ) + + assert json.loads(descriptor.read_text(encoding="utf-8"))["hf_id"] == str( + model_directory.resolve() + ) + + +@pytest.mark.parametrize("artifact_path", ["../outside.npy", "absolute"]) +def test_accuracy_rejects_candidate_artifacts_outside_the_cell_directory( + tmp_path: Path, monkeypatch, artifact_path: str +) -> None: + case = _example_case(tmp_path) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + outside = tmp_path / "outside.npy" + reported_path = str(outside.resolve()) if artifact_path == "absolute" else artifact_path + + monkeypatch.setattr( + qualification_accuracy, + "require_candidate", + lambda _context: (tmp_path / "worker", tmp_path / "runtime"), + ) + monkeypatch.setattr( + qualification_accuracy, + "write_model_descriptor", + lambda *_args, **_kwargs: tmp_path / "model.json", + ) + monkeypatch.setattr( + qualification_accuracy, + "prepare_bundle", + lambda *_args, **_kwargs: tmp_path / "model.bundle", + ) + def candidate(command, *_args, **_kwargs): + candidate_output = Path(command[command.index("--output") + 1]) + candidate_output.mkdir(parents=True) + (candidate_output / "result.json").write_text( + json.dumps( + { + "cells": [ + { + "status": "completed", + "artifact_dir": "artifacts/case", + "output_summary": {"mask_artifact": reported_path}, + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(qualification_accuracy, "run_command", candidate) + + output = tmp_path / "qualification" + output.mkdir() + with pytest.raises(QualificationError, match="unsafe artifact path"): + qualification_accuracy._candidate_outputs( + case, + context, + output, + "segment", + [{"request": {"image_path": "/data/image.jpg"}}], + ) + + +@pytest.mark.parametrize( + ("task", "public_request", "expected_inputs", "expected_controls"), + [ + ( + "stereo_disparity", + { + "left_image_path": "/data/left.png", + "right_image_path": "/data/right.png", + "height": 700, + "width": 700, + }, + {"left_image": "/data/left.png", "right_image": "/data/right.png"}, + {"height": 700, "width": 700}, + ), + ( + "robot_control", + {"image_path": "/data/observation.png", "state_path": "/data/state.f32"}, + {"image": "/data/observation.png", "state": "/data/state.f32"}, + {}, + ), + ], +) +def test_candidate_descriptor_uses_manifest_input_contract( + tmp_path: Path, + task: str, + public_request: dict[str, object], + expected_inputs: dict[str, str], + expected_controls: dict[str, object], +) -> None: + case = QualificationCase( + kind="performance", + model="example-model", + family="example", + name="case", + benchmark="example_performance", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": task, + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, + ) + + descriptor = write_model_descriptor(case, tmp_path, public_request) + testcase = json.loads(descriptor.read_text(encoding="utf-8"))["testcases"][0] - assert '"hf_id": "openai-community/gpt2"' in value - assert '"task": "text_generation"' in value - assert '"max_sequence_length": 1024' in value - assert "tests/manifests" not in value + assert testcase["inputs"] == expected_inputs + for name, value in expected_controls.items(): + assert testcase[name] == value + assert not (set(public_request) & set(testcase["inputs"])) def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> None: @@ -122,17 +2374,167 @@ def test_internal_subprocesses_can_import_repository_packages(tmp_path: Path) -> assert completed.returncode == 0, completed.stderr -def test_bundle_preparation_uses_the_selected_runtime( +def test_family_reference_environment_inherits_parent_venv_packages( tmp_path: Path, monkeypatch ) -> None: - case = next( - case - for case in select(discover(REPOSITORY), ["gpt2-125m"]) - if case.kind == "accuracy" + requirements = tmp_path / "requirements.txt" + requirements.write_text("transformers==4.46.3\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="custom-environment-model", + family="example", + name="continuation", + benchmark="mmlu_continuation", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=requirements, + reference_build_isolation=False, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + parent_packages = tmp_path / "parent-venv/site-packages" + parent_packages.mkdir(parents=True) + commands = [] + environments = [] + timeouts = [] + + def complete(command, *_args, **_kwargs): + commands.append(command) + environments.append(_kwargs.get("env")) + timeouts.append(_kwargs.get("timeout")) + if command[1:3] == ["-m", "venv"]: + environment = Path(command[-1]) + (environment / "bin").mkdir(parents=True) + (environment / "bin/python").write_text("", encoding="utf-8") + child_packages = environment / ( + f"lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages" + ) + child_packages.mkdir(parents=True) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr("qualification_tests.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("site.getsitepackages", lambda: [str(parent_packages)]) + + python = reference_python(case, context) + + inherited = python.parents[1] / ( + f"lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages/" + "trtmc-parent-environment.pth" + ) + assert inherited.read_text(encoding="utf-8") == f"{parent_packages.resolve()}\n" + pip_command = next(command for command in commands if command[1:3] == ["-m", "pip"]) + assert "--no-build-isolation" in pip_command + pip_index = commands.index(pip_command) + assert environments[pip_index]["MAX_JOBS"] == "4" + assert timeouts[pip_index] == 7200 + + +def test_family_environment_hook_runs_after_requirements_install( + tmp_path: Path, monkeypatch +) -> None: + requirements = tmp_path / "requirements.txt" + requirements.write_text("example==1\n", encoding="utf-8") + hook = tmp_path / "prepare_environment.py" + hook.write_text("# family hook\n", encoding="utf-8") + case = QualificationCase( + kind="accuracy", + model="custom-environment-model", + family="example", + name="continuation", + benchmark="mmlu_continuation", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "build": {}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=requirements, + environment_hook=hook, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=False, + verbose=False, + ) + commands: list[list[str]] = [] + parent_packages = tmp_path / "parent/site-packages" + parent_packages.mkdir(parents=True) + + def complete(command, *_args, **_kwargs): + commands.append(list(command)) + if command[1:3] == ["-m", "venv"]: + environment = Path(command[-1]) + (environment / "bin").mkdir(parents=True) + (environment / "bin/python").write_text("", encoding="utf-8") + child = environment / ( + f"lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages" + ) + child.mkdir(parents=True) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr("qualification_tests.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("site.getsitepackages", lambda: [str(parent_packages)]) + + python = reference_python(case, context) + + assert commands[-1] == [str(python), str(hook)] + + +def test_bundle_preparation_uses_the_selected_runtime(tmp_path: Path, monkeypatch) -> None: + case = QualificationCase( + kind="accuracy", + model="example-model", + family="example", + name="continuation", + benchmark="example_accuracy", + candidate={ + "family": "example", + "checkpoint": "example/model", + "task": "text_generation", + "precision": "fp16", + "bundle": "example.bundle", + "build": {"max_sequence_length": 16}, + }, + values={}, + source=tmp_path / "example.yaml", + reference_requirements=None, ) runtime_root = tmp_path / "runtime" runtime_root.mkdir() - bundle = tmp_path / "gpt2.bundle" + bundle = tmp_path / "example.bundle" bundle.write_bytes(b"bundle") descriptor = tmp_path / "candidate-model.json" descriptor.write_text("{}\n", encoding="utf-8") @@ -162,9 +2564,10 @@ def complete(command, *_args, **_kwargs): stderr="", ) - monkeypatch.setattr("tools.benchmark_qualification.runtime.run_command", complete) + monkeypatch.setattr("qualification_tests.benchmark_qualification.runtime.run_command", complete) assert prepare_bundle(case, context, tmp_path, descriptor) == bundle + assert captured[0] == str(context.trtmc_bench) assert captured[captured.index("--runtime-root") + 1] == str(runtime_root) @@ -246,3 +2649,290 @@ def download(_url, destination): assert resolved.path == tmp_path / "cache/public/public.jsonl" assert resolved.path.read_bytes() == payload assert resolve_dataset(definition, context).path == resolved.path + + +def test_family_dataset_stays_with_its_discovered_model_profile(tmp_path: Path) -> None: + profile = tmp_path / "families/stereo/tests/benchmark/stereo.yaml" + dataset = tmp_path / "families/stereo/tests/data/pairs.json" + profile.parent.mkdir(parents=True) + dataset.parent.mkdir(parents=True) + dataset.write_text('{"requests": []}\n', encoding="utf-8") + digest = hashlib.sha256(dataset.read_bytes()).hexdigest() + context = RuntimeContext( + repository=tmp_path, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "cache", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + definition = { + "dataset": { + "id": "family-pairs", + "sha256": digest, + "source": {"mode": "family", "path": "../data/pairs.json"}, + } + } + + resolved = resolve_dataset(definition, context, profile) + + assert resolved.path == dataset + assert resolved.receipt()["source_mode"] == "family" + definition["dataset"]["source"]["path"] = "../../../../outside.json" + with pytest.raises(QualificationError, match="inside its family"): + resolve_dataset(definition, context, profile) + + +def test_stereo_accuracy_compares_complete_disparity_artifacts(tmp_path: Path, monkeypatch) -> None: + profile = tmp_path / "families/stereo/tests/benchmark/stereo.yaml" + reference_script = profile.parent / "reference.py" + reference_script.parent.mkdir(parents=True) + reference_script.write_text("# family reference\n", encoding="utf-8") + data = tmp_path / "data" + data.mkdir() + for name in ("left.png", "right.png"): + (data / name).write_bytes(b"fixture") + dataset_path = data / "pairs.json" + dataset_path.write_text( + json.dumps( + { + "requests": [ + { + "id": "pair", + "left_image": "left.png", + "right_image": "right.png", + } + ] + } + ), + encoding="utf-8", + ) + dataset = Dataset("stereo", dataset_path, "provided", "digest") + case = QualificationCase( + kind="accuracy", + model="stereo-model", + family="stereo", + name="parity", + benchmark="stereo_disparity_parity", + candidate={ + "family": "stereo", + "checkpoint": "example/stereo", + "task": "stereo_disparity", + "precision": "fp16", + "build": {}, + }, + values={ + "samples": 1, + "request": {"height": 1, "width": 2}, + "reference": {"command": "reference.py", "precision": "fp16"}, + "gate": { + "min_cosine": 0.99, + "max_mean_abs_error": 0.2, + "max_bad_2px_fraction": 0.0, + "min_sample_pass_rate": 1.0, + }, + }, + source=profile, + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data-root", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + reference_artifact = tmp_path / "reference.f32" + reference_artifact.write_bytes(array("f", [1.0, 2.0]).tobytes()) + candidate_artifact = tmp_path / "candidate.f32" + candidate_artifact.write_bytes(array("f", [1.1, 2.1]).tobytes()) + + def run_reference(command, *_args, **_kwargs): + output = Path(command[command.index("--output") + 1]) + output.write_text( + json.dumps( + { + "samples": [ + { + "sample_id": "pair", + "height": 1, + "width": 2, + "element_count": 2, + "disparity_artifact": str(reference_artifact), + } + ] + } + ), + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "stereo_disparity_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "reference_python", lambda *_args: Path(sys.executable) + ) + monkeypatch.setattr(qualification_accuracy, "run_command", run_reference) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ( + [ + { + "height": 1, + "width": 2, + "element_count": 2, + "disparity_artifact": str(candidate_artifact), + } + ], + tmp_path / "stereo.bundle", + ), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["sample_pass_rate"] == 1.0 + assert result["metrics"]["max_mean_abs_error"] == pytest.approx(0.1) + + +def test_metric_geometry_accuracy_compares_complete_task_artifacts( + tmp_path: Path, monkeypatch +) -> None: + profile = tmp_path / "families/example/tests/benchmark/example-geometry.yaml" + profile.parent.mkdir(parents=True) + image = tmp_path / "data/image.jpeg" + image.parent.mkdir() + image.write_bytes(b"fixture") + dataset_path = image.parent / "dataset.json" + dataset_path.write_text( + json.dumps({"requests": [{"id": "image", "image": "image.jpeg"}]}), + encoding="utf-8", + ) + dataset = Dataset("geometry", dataset_path, "provided", "digest") + + def geometry(root: Path) -> dict[str, object]: + root.mkdir() + (root / "points.f32").write_bytes( + array("f", [1.0, 2.0, 3.0, float("inf"), float("inf"), float("inf")]).tobytes() + ) + (root / "depth.f32").write_bytes(array("f", [3.0, float("inf")]).tobytes()) + (root / "mask.u8").write_bytes(bytes((1, 0))) + return { + "geometry_images": 1, + "geometry_pixels": 2, + "height": 1, + "width": 2, + "point_shape": [1, 2, 3], + "valid_pixels": 1, + "normalized_intrinsics": [ + [1.0, 0.0, 0.5], + [0.0, 1.0, 0.5], + [0.0, 0.0, 1.0], + ], + "units": "meters", + "camera_axes": ["right", "down", "forward"], + "intrinsics_coordinates": "normalized_uv", + "points_artifact": str(root / "points.f32"), + "depth_artifact": str(root / "depth.f32"), + "valid_mask_artifact": str(root / "mask.u8"), + } + + candidate = geometry(tmp_path / "candidate") + reference = geometry(tmp_path / "reference") + thresholds = { + "mask_iou": 0.999, + "depth_absrel_mean": 0.005, + "depth_rel_l2": 0.02, + "points_rel_l2": 0.02, + "points_cosine": 0.99999, + "intrinsics_max_relative_error": 0.002, + "point_depth_consistency": 0.00001, + "min_sample_pass_rate": 1.0, + } + case = QualificationCase( + kind="accuracy", + model="example-geometry", + family="example", + name="geometry", + benchmark="metric_geometry_parity", + candidate={ + "family": "example", + "checkpoint": "example/geometry", + "task": "monocular_geometry", + "precision": "fp32", + "build": {}, + }, + values={ + "samples": 1, + "request": {"num_tokens": 1800}, + "reference": {"command": "reference.py", "precision": "fp32"}, + "gate": thresholds, + }, + source=profile, + reference_requirements=None, + ) + context = RuntimeContext( + repository=REPOSITORY, + artifacts=tmp_path / "artifacts", + data_root=tmp_path / "data-root", + environment_root=tmp_path / "envs", + bundle_cache=tmp_path / "bundles", + bundle_roots=(), + runtime_root=None, + trtmc_bench=tmp_path / "trtmc-bench", + worker=None, + datasets={}, + reference_pythons={}, + no_build=True, + verbose=False, + ) + monkeypatch.setattr( + qualification_accuracy, + "load_benchmark", + lambda *_args: {"metric": {"name": "metric_geometry_parity"}}, + ) + monkeypatch.setattr(qualification_accuracy, "resolve_dataset", lambda *_args: dataset) + monkeypatch.setattr( + qualification_accuracy, "_family_image_reference", lambda *_args: [reference] + ) + monkeypatch.setattr( + qualification_accuracy, + "_candidate_outputs", + lambda *_args: ([candidate], tmp_path / "example.bundle"), + ) + + result = qualification_accuracy.run_accuracy(case, context) + + assert result["status"] == "passed" + assert result["metrics"]["samples"] == 1 + assert result["metrics"]["sample_pass_rate"] == 1.0 + assert result["metrics"]["mask_iou"] == 1.0 + assert result["metrics"]["points_cosine"] == pytest.approx(1.0) + for name in ( + "depth_absrel_mean", + "depth_rel_l2", + "points_rel_l2", + "intrinsics_max_relative_error", + "point_depth_consistency", + ): + assert result["metrics"][name] == 0.0 diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index 47de6f7147..a8fa291693 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -31,7 +31,7 @@ from tools.ci.pipeline import CiPipeline from tools.ci.process import CiError from tools.ci.quality import SourceQualityChecks, UnitTestRunner -from tools import perf_matrix +from qualification_tests.benchmark_qualification.performance import matrix as perf_matrix class RecordingContext: diff --git a/website/docs/reference/benchmarking.md b/website/docs/reference/benchmarking.md index a90277071b..521ab6fde6 100644 --- a/website/docs/reference/benchmarking.md +++ b/website/docs/reference/benchmarking.md @@ -9,12 +9,10 @@ does not register model families or import family implementation code. ## Single-model benchmark -Install the optional benchmark dependencies, then select an installed runtime -root containing the core runtime, TensorRT backend, and required family DSOs: +Install TRTMC, then select an installed runtime root containing the core +runtime, TensorRT backend, and required family DSOs: ```bash -python -m pip install -r apps/benchmark/performance/requirements.txt - trtmc-bench list models trtmc-bench run \ --model distilgpt2 \ @@ -87,7 +85,10 @@ reference within five percent of candidate p50 is considered equivalent. Candidate or reference execution failures are operational failures, not slow performance results. -## Release performance matrix +## Internal release performance matrix + +This repository-only workflow is CI qualification. It is separate from the +installed `trtmc-bench` user interface above and is not included in the wheel. The matrix coordinates candidate and reference runs without introducing a second model registry: @@ -97,19 +98,19 @@ export TRTMC_PERF_WORKER=/opt/trtmc/bin/trtmc_benchmark_worker export TRTMC_PERF_RUNTIME_ROOT=/opt/trtmc/lib export TRTMC_PERF_BUNDLE_CACHE=/data/trtmc-bundles -python3 tools/perf_matrix.py check \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml +python3 -m qualification_tests.benchmark_qualification.performance check \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml -python3 tools/perf_matrix.py prepare \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ +python3 -m qualification_tests.benchmark_qualification.performance prepare \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ --entry gpt2.generate \ --output artifacts/perf/bundle-preparation.json -python3 tools/perf_matrix.py run \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ +python3 -m qualification_tests.benchmark_qualification.performance run \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ --entry gpt2.generate ``` @@ -117,8 +118,8 @@ Preparation is deliberately separate and untimed. Resume or regenerate a report from stored observations with: ```bash -python3 tools/perf_matrix.py resume artifacts/perf/ -python3 tools/perf_matrix.py report artifacts/perf/ +python3 -m qualification_tests.benchmark_qualification.performance resume artifacts/perf/ +python3 -m qualification_tests.benchmark_qualification.performance report artifacts/perf/ ``` The release YAML owns model, testcase, operation, measurement, reference, and diff --git a/website/docs/tutorials/advanced/validation-and-benchmarking.md b/website/docs/tutorials/advanced/validation-and-benchmarking.md index da01dd405a..8f42503f6b 100644 --- a/website/docs/tutorials/advanced/validation-and-benchmarking.md +++ b/website/docs/tutorials/advanced/validation-and-benchmarking.md @@ -51,13 +51,13 @@ trtmc-bench run \ For release comparisons, validate and run the checked-in matrix: ```bash -python3 tools/perf_matrix.py check \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml +python3 -m qualification_tests.benchmark_qualification.performance check \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml -python3 tools/perf_matrix.py run \ - apps/benchmark/performance/release.yaml \ - --environment apps/benchmark/performance/environments/gb300.yaml \ +python3 -m qualification_tests.benchmark_qualification.performance run \ + qualification_tests/benchmark_qualification/performance/config/release.yaml \ + --environment qualification_tests/benchmark_qualification/performance/config/environments/gb300.yaml \ --entry gpt2.generate ```